carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: no_sort_columns
Date Wed, 05 Apr 2017 13:46:35 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/12-dev 3f19978c0 -> dccd2ad0d


no_sort_columns

fix comments


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/dbd0cdd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/dbd0cdd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/dbd0cdd9

Branch: refs/heads/12-dev
Commit: dbd0cdd900e491d1e780ff6b84dbbabf781cf9c3
Parents: 3f19978
Author: QiangCai <qiangcai@qq.com>
Authored: Mon Apr 3 00:42:55 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Wed Apr 5 21:44:56 2017 +0800

----------------------------------------------------------------------
 .../core/datastore/block/SegmentProperties.java |  22 ++-
 .../impl/btree/BTreeDataRefNodeFinder.java      |  24 +--
 .../core/metadata/schema/table/CarbonTable.java |   4 -
 .../scan/filter/FilterExpressionProcessor.java  |   4 +-
 .../carbondata/core/scan/filter/FilterUtil.java |   8 +-
 ...velRangeLessThanEqualFilterExecuterImpl.java |   2 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |   2 +-
 .../AbstractDetailQueryResultIterator.java      |   4 +-
 .../apache/carbondata/core/util/CarbonUtil.java |  18 +++
 .../impl/btree/BTreeBlockFinderTest.java        |  10 +-
 .../carbondata/hadoop/CarbonInputFormat.java    |   4 +-
 .../internal/index/impl/InMemoryBTreeIndex.java |   4 +-
 .../hadoop/util/BlockLevelTraverser.java        |   4 +-
 .../testsuite/sortcolumns/TestSortColumns.scala |  79 ++++++++++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  11 +-
 .../org/apache/spark/sql/TableCreator.scala     |  11 +-
 .../newflow/CarbonDataLoadConfiguration.java    |   4 +
 .../newflow/DataLoadProcessBuilder.java         |  22 +--
 .../newflow/steps/NoSortProcessorStepImpl.java  | 153 +++++++++++++++++++
 .../store/CarbonFactDataHandlerModel.java       |   7 +
 20 files changed, 350 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 3bef423..80785e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
   private KeyGenerator dimensionKeyGenerator;
 
   /**
+   * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+   * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+   * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+   * than dimensionKeyGenerator.
+   */
+  private KeyGenerator sortColumnsGenerator;
+
+  /**
    * list of dimension present in the block
    */
   private List<CarbonDimension> dimensions;
@@ -522,6 +530,14 @@ public class SegmentProperties {
     int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
     // create a key generator
     this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+    if (this.getNumberOfDictSortColumns() == bitLength.length) {
+      this.sortColumnsGenerator = this.dimensionKeyGenerator;
+    } else {
+      int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+      int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+      System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+      this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+    }
     this.fixedLengthKeySplitter =
         new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
     // get the size of each value in file block
@@ -647,6 +663,10 @@ public class SegmentProperties {
     return dimensionKeyGenerator;
   }
 
+  public KeyGenerator getSortColumnsGenerator() {
+    return sortColumnsGenerator;
+  }
+
   /**
    * @return the dimensions
    */
@@ -817,6 +837,6 @@ public class SegmentProperties {
   }
 
   public int getNumberOfDictSortColumns() {
-    return this.numberOfSortColumns - this.numberOfNoDictionaryDimension;
+    return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
   private int[] eachColumnValueSize;
 
   /**
-   * this will be used during search for no dictionary column
+   * the number of no dictionary columns in SORT_COLUMNS
    */
-  private int numberOfNoDictionaryColumns;
+  private int numberOfNoDictSortColumns;
 
-  public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
-    this.eachColumnValueSize = eachColumnValueSize;
+  /**
+   * the number of columns in SORT_COLUMNS
+   */
+  private int numberOfSortColumns;
 
-    for (int i = 0; i < eachColumnValueSize.length; i++) {
-      if (eachColumnValueSize[i] == -1) {
-        numberOfNoDictionaryColumns++;
-      }
-    }
+  public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+      int numberOfNoDictSortColumns) {
+    this.eachColumnValueSize = eachColumnValueSize;
+    this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+    this.numberOfSortColumns = numberOfSortColumns;
   }
 
   /**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
     int dictionaryKeyOffset = 0;
     int nonDictionaryKeyOffset = 0;
     int compareResult = 0;
-    int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+    int processedNoDictionaryColumn = numberOfNoDictSortColumns;
     ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
     ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
     int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
     int firstNoDcitionaryLength = 0;
     int secondNodeDictionaryLength = 0;
 
-    for (int i = 0; i < eachColumnValueSize.length; i++) {
+    for (int i = 0; i < numberOfSortColumns; i++) {
 
       if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
         compareResult = ByteUtil.UnsafeComparer.INSTANCE

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 1271b6a..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -656,10 +656,6 @@ public class CarbonTable implements Serializable {
     tableMeasuresMap.put(tableName, visibleMeasures);
   }
 
-  public boolean isSortByColumns() {
-    return numberOfSortColumns > 0;
-  }
-
   public int getNumberOfSortColumns() {
     return numberOfSortColumns;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
             + searchEndKey.getNoDictionaryKeys());
     long startTimeInMillis = System.currentTimeMillis();
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
-        tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+        tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+        tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+        tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
     DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
     DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
     FilterExecuter filterExecuter =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index bb26ea5..b116d9a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1065,7 +1065,7 @@ public final class FilterUtil {
     }
     IndexKey endIndexKey;
     byte[] dictionaryendMdkey =
-        segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+        segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
     byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
     endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
     return endIndexKey;
@@ -1112,7 +1112,7 @@ public final class FilterUtil {
     IndexKey startIndexKey;
     long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
     byte[] dictionaryStartMdkey =
-        segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+        segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
     byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
 
     startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1261,11 +1261,11 @@ public final class FilterUtil {
     }
 
     searchStartKey = FilterUtil
-        .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+        .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
             FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
 
     searchEndKey = FilterUtil
-        .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+        .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
             FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
     listOfStartEndKeys.add(searchStartKey);
     listOfStartEndKeys.add(searchEndKey);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 306becf..bd83eec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
       CarbonDimension currentBlockDimension =
           segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
       defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
-          this.segmentProperties.getDimensionKeyGenerator());
+          this.segmentProperties.getSortColumnsGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
         && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index f2a49d9..a439228 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
       CarbonDimension currentBlockDimension =
           segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
       defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
-          this.segmentProperties.getDimensionKeyGenerator());
+          this.segmentProperties.getSortColumnsGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
         && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
 
   private void intialiseInfos() {
     for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
-      DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+      DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+          blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+          blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
       DataRefNode startDataBlock = finder
           .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
       while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index a442087..7c63f28 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
   }
 
   /**
+   * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+   * @param parseStr
+   * @return
+   */
+  public static String unquoteChar(String parseStr) {
+    if (parseStr == null) {
+      return null;
+    }
+    if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+      return parseStr.substring(1, parseStr.length() - 1);
+    } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+      return parseStr.substring(1, parseStr.length() - 1);
+    } else {
+      return parseStr;
+    }
+  }
+
+  /**
    * special char delimiter Converter
    *
    * @param delimiter

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index a4adf92..245ff48 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
     ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
     buffer.rewind();
     buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
     ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
     buffer.rewind();
     buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =
@@ -160,7 +160,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index b330f12..540a69c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -578,7 +578,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
 
       // Add all blocks of btree into result
       DataRefNodeFinder blockFinder =
-          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+              segmentProperties.getNumberOfSortColumns(),
+              segmentProperties.getNumberOfNoDictSortColumns());
       DataRefNode startBlock =
           blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
       DataRefNode endBlock =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
 
       // Add all blocks of btree into result
       DataRefNodeFinder blockFinder =
-          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+              segmentProperties.getNumberOfSortColumns(),
+              segmentProperties.getNumberOfNoDictSortColumns());
       DataRefNode startBlock =
           blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
       DataRefNode endBlock =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
           FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
 
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
-        abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+        abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+        abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+        abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
     DataRefNode currentBlock =
         blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 3b28453..88b0421 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -154,6 +154,79 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
     checkAnswer(sql("select * from sorttable6 where empname = 'madhan'"), sql("select * from origintable1 where empname = 'madhan'"))
   }
 
+  test("unsorted table creation, query data loading with heap and safe sort config") {
+    try {
+      setLoadingProperties("false", "false", "false")
+      sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+  test("unsorted table creation, query and data loading with heap and unsafe sort config") {
+    try {
+      setLoadingProperties("false", "true", "false")
+      sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+  test("unsorted table creation, query and loading with heap and inmemory sort config") {
+    try {
+      setLoadingProperties("false", "false", "true")
+      sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+  test("unsorted table creation, query and data loading with offheap and safe sort config") {
+    try {
+      setLoadingProperties("true", "false", "false")
+      sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+  test("unsorted table creation, query and data loading with offheap and unsafe sort config") {
+    try {
+      setLoadingProperties("true", "true", "false")
+      sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+  test("unsorted table creation, query and data loading with offheap and inmemory sort config") {
+    try {
+      setLoadingProperties("true", "false", "true")
+      sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+      sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+    } finally {
+      defaultLoadingProperties
+    }
+  }
+
+
   override def afterAll = {
     dropTable
   }
@@ -172,6 +245,12 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
     sql("drop table if exists sorttable4_heap_inmemory")
     sql("drop table if exists sorttable5")
     sql("drop table if exists sorttable6")
+    sql("drop table if exists unsortedtable_offheap_safe")
+    sql("drop table if exists unsortedtable_offheap_unsafe")
+    sql("drop table if exists unsortedtable_offheap_inmemory")
+    sql("drop table if exists unsortedtable_heap_safe")
+    sql("drop table if exists unsortedtable_heap_unsafe")
+    sql("drop table if exists unsortedtable_heap_inmemory")
   }
 
   def setLoadingProperties(offheap: String, unsafe: String, useBatch: String): Unit = {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 4e95073..d5c1bcd 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.command._
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
 import org.apache.carbondata.processing.constants.LoggerAction
 import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
 import org.apache.carbondata.spark.util.CommonUtil
@@ -490,8 +490,13 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
     // All columns in sortkey should be there in create table cols
     val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
     var sortKeyDimsTmp: Seq[String] = Seq[String]()
-    if (sortKeyOption.isDefined) {
-      var sortKey = sortKeyOption.get.split(',').map(_.trim)
+    val sortKeyString: String = if (sortKeyOption.isDefined) {
+      CarbonUtil.unquoteChar(sortKeyOption.get) trim
+    } else {
+      ""
+    }
+    if (!sortKeyString.isEmpty) {
+      val sortKey = sortKeyString.split(',').map(_.trim)
       sortKey.foreach { column =>
         if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
           val errormsg = "sort_columns: " + column +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index 9321706..9e14f16 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.command._
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
 import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
 import org.apache.carbondata.spark.util.CommonUtil
 
@@ -62,8 +62,13 @@ object TableCreator {
     // All columns in sortkey should be there in create table cols
     val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
     var sortKeyDimsTmp: Seq[String] = Seq[String]()
-    if (sortKeyOption.isDefined) {
-      var sortKey = sortKeyOption.get.split(',').map(_.trim)
+    val sortKeyString: String = if (sortKeyOption.isDefined) {
+      CarbonUtil.unquoteChar(sortKeyOption.get) trim
+    } else {
+      ""
+    }
+    if (!sortKeyString.isEmpty) {
+      val sortKey = sortKeyString.split(',').map(_.trim)
       sortKey.foreach { column =>
         if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
           val errormsg = "sort_columns: " + column +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 407ac2f..4a6c421 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -133,6 +133,10 @@ public class CarbonDataLoadConfiguration {
     return this.numberOfSortColumns;
   }
 
+  public boolean isSortTable() {
+    return this.numberOfSortColumns > 0;
+  }
+
   public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
     this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 1932888..e5f6a5f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWith
 import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
 import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
 import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
+import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
 import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
 import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
@@ -76,9 +77,10 @@ public final class DataLoadProcessBuilder {
     // data types and configurations.
     AbstractDataLoadProcessorStep converterProcessorStep =
         new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
-    // 3. Sorts the data which are part of key (all dimensions except complex types)
-    AbstractDataLoadProcessorStep sortProcessorStep =
-        new SortProcessorStepImpl(configuration, converterProcessorStep);
+    // 3. Sorts the data by SortColumn or not
+    AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+        new SortProcessorStepImpl(configuration, converterProcessorStep) :
+        new NoSortProcessorStepImpl(configuration, converterProcessorStep);
     // 4. Writes the sorted data in carbondata format.
     AbstractDataLoadProcessorStep writerProcessorStep =
         new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -94,9 +96,10 @@ public final class DataLoadProcessBuilder {
     // data types and configurations.
     AbstractDataLoadProcessorStep converterProcessorStep =
         new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
-    // 3. Sorts the data which are part of key (all dimensions except complex types)
-    AbstractDataLoadProcessorStep sortProcessorStep =
-        new SortProcessorStepImpl(configuration, converterProcessorStep);
+    // 3. Sorts the data by SortColumn or not
+    AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+        new SortProcessorStepImpl(configuration, converterProcessorStep) :
+        new NoSortProcessorStepImpl(configuration, converterProcessorStep);
     // 4. Writes the sorted data in carbondata format.
     AbstractDataLoadProcessorStep writerProcessorStep =
         new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -112,9 +115,10 @@ public final class DataLoadProcessBuilder {
     // data types and configurations.
     AbstractDataLoadProcessorStep converterProcessorStep =
         new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
-    // 3. Sorts the data which are part of key (all dimensions except complex types)
-    AbstractDataLoadProcessorStep sortProcessorStep =
-        new SortProcessorStepImpl(configuration, converterProcessorStep);
+    // 3. Sorts the data by SortColumn or not
+    AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+        new SortProcessorStepImpl(configuration, converterProcessorStep) :
+        new NoSortProcessorStepImpl(configuration, converterProcessorStep);
     // 4. Writes the sorted data in carbondata format.
     AbstractDataLoadProcessorStep writerProcessorStep =
         new DataWriterProcessorStepImpl(configuration, sortProcessorStep);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
new file mode 100644
index 0000000..d44f954
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.carbondata.processing.util.NonDictionaryUtil;
+
+/**
+ * if the table doesn't have sort_columns, just convert row format.
+ */
+public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+  private int dimensionCount;
+
+  private int dimensionWithComplexCount;
+
+  private int noDictCount;
+
+  private int measureCount;
+
+  private boolean[] isNoDictionaryDimensionColumn;
+
+  private char[] aggType;
+
+  public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+      AbstractDataLoadProcessorStep child) {
+    super(configuration, child);
+    this.dimensionWithComplexCount = configuration.getDimensionCount();
+    this.noDictCount =
+        configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+    this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
+    this.measureCount = configuration.getMeasureCount();
+    this.isNoDictionaryDimensionColumn =
+        CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+    this.aggType = CarbonDataProcessorUtil
+        .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+  }
+
+  @Override public DataField[] getOutput() {
+    return child.getOutput();
+  }
+
+  @Override public void initialize() throws IOException {
+    child.initialize();
+  }
+
+  /**
+   * convert input CarbonRow to output CarbonRow
+   * e.g. There is a table as following,
+   * the number of dictionary dimensions is a,
+   * the number of no-dictionary dimensions is b,
+   * the number of complex dimensions is c,
+   * the number of measures is d.
+   * input CarbonRow format:  the length of Object[] data is a+b+c+d, the number of all columns.
+   * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+   * | Part                     | Object item                    | describe                 |
+   * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+   * | Object[0 ~ a+b-1]        | Integer, byte[], Integer, ...  | dict + no dict dimensions|
+   * ----------------------------------------------------------------------------------------
+   * | Object[a+b ~ a+b+c-1]    | byte[], byte[], ...            | complex dimensions       |
+   * ----------------------------------------------------------------------------------------
+   * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ...               | measures                 |
+   * ----------------------------------------------------------------------------------------
+   * output CarbonRow format: the length of object[] data is 3.
+   * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+   * | Part                     | Object item                    | describe                 |
+   * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+   * | Object[0]                | int[a]                         | dict dimension array     |
+   * ----------------------------------------------------------------------------------------
+   * | Object[1]                | byte[b+c][]                    | no dict + complex dim    |
+   * ----------------------------------------------------------------------------------------
+   * | Object[2]                | Object[d]                      | measures                 |
+   * ----------------------------------------------------------------------------------------
+   * @param row
+   * @return
+   */
+  @Override protected CarbonRow processRow(CarbonRow row) {
+    int dictIndex = 0;
+    int nonDicIndex = 0;
+    int[] dim = new int[this.dimensionCount];
+    byte[][] nonDicArray = new byte[this.noDictCount][];
+    Object[] measures = new Object[this.measureCount];
+    // read dimension values
+    int dimCount = 0;
+    for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+      if (isNoDictionaryDimensionColumn[dimCount]) {
+        nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+      } else {
+        dim[dictIndex++] = (int) row.getObject(dimCount);
+      }
+    }
+
+    for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+      nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+    }
+
+    // measure values
+    for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
+      Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
+      if (null != value) {
+        if (aggType[mesCount] == CarbonCommonConstants.SUM_COUNT_VALUE_MEASURE) {
+          measures[mesCount] = value;
+        } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
+          measures[mesCount] = value;
+        } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+          BigDecimal val = (BigDecimal) value;
+          measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
+        }
+      } else {
+        measures[mesCount] = null;
+      }
+    }
+    // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+    Object[] holder = new Object[3];
+    NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
+    //return out row
+    return new CarbonRow(holder);
+  }
+
+  @Override
+  public void close() {
+    if (!closed) {
+      super.close();
+    }
+  }
+
+  @Override protected String getStepName() {
+    return "No Sort Processor";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index ffd23a2..e746514 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -199,6 +199,13 @@ public class CarbonFactDataHandlerModel {
         CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
 
     int[] dimLensWithComplex = configuration.getCardinalityFinder().getCardinality();
+    if (!configuration.isSortTable()) {
+      for (int i = 0; i < dimLensWithComplex.length; i++) {
+        if (dimLensWithComplex[i] != 0) {
+          dimLensWithComplex[i] = Integer.MAX_VALUE;
+        }
+      }
+    }
     List<Integer> dimsLenList = new ArrayList<Integer>();
     for (int eachDimLen : dimLensWithComplex) {
       if (eachDimLen != 0) dimsLenList.add(eachDimLen);


Mime
View raw message