carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [48/50] [abbrv] incubator-carbondata git commit: Merge remote-tracking branch 'carbon_master/master' into apache/master
Date Thu, 30 Jun 2016 17:42:35 GMT
Merge remote-tracking branch 'carbon_master/master' into apache/master

Conflicts:
	README.md
	core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
	core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
	core/src/main/java/org/carbondata/core/util/CarbonUtil.java
	core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
	core/src/main/java/org/carbondata/query/aggregator/impl/CustomAggregatorHelper.java
	core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
	core/src/main/java/org/carbondata/query/carbon/processor/impl/AggregateQueryBlockProcessor.java
	core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java
	core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
	core/src/main/java/org/carbondata/scan/expression/Expression.java
	core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
	core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
	core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
	core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
	core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/ListExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
	core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
	core/src/main/java/org/carbondata/scan/expression/exception/FilterUnsupportedException.java
	core/src/main/java/org/carbondata/scan/expression/logical/AndExpression.java
	core/src/main/java/org/carbondata/scan/expression/logical/NotExpression.java
	core/src/main/java/org/carbondata/scan/expression/logical/OrExpression.java
	core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
	core/src/main/java/org/carbondata/scan/filter/executer/ColGroupFilterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
	core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
	core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
	core/src/main/java/org/carbondata/scan/filter/resolver/FilterResolverIntf.java
	core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
	core/src/main/java/org/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java
	core/src/main/java/org/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
	core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
	core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
	core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
	core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
	integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase1.scala
	integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase2.scala
	integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
	integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
	integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
	integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
	integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
	integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
	integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
	integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
	integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
	integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
	processing/src/main/java/org/carbondata/processing/store/CarbonFactDataHandlerModel.java
	processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupDataHolder.java
	processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupMinMax.java
	processing/src/main/java/org/carbondata/processing/store/writer/AbstractFactDataWriter.java
	processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f722186
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f722186
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f722186

Branch: refs/heads/master
Commit: 7f72218602926ca5f6629825df7b8f70ab1c6aa6
Parents: e9bbf75 d44c9e7
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Thu Jun 30 17:41:47 2016 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu Jun 30 17:41:47 2016 +0530

----------------------------------------------------------------------
 common/pom.xml                                  |    8 +-
 .../common/logging/impl/StandardLogService.java |   15 +-
 conf/carbon.properties.template                 |   98 +
 .../common/ext/ColumnUniqueIdGenerator.java     |   41 +
 .../common/ext/DictionaryFactory.java           |  119 +
 .../org/carbondata/common/ext/PathFactory.java  |   48 +
 .../common/factory/CarbonCommonFactory.java     |   54 +
 .../dictionary/AbstractDictionaryCache.java     |   43 +-
 .../cache/dictionary/ColumnDictionaryInfo.java  |    5 +-
 .../cache/dictionary/DictionaryCacheLoader.java |    7 +-
 .../dictionary/DictionaryCacheLoaderImpl.java   |   28 +-
 .../DictionaryColumnUniqueIdentifier.java       |    9 +-
 .../dictionary/ForwardDictionaryCache.java      |   11 +-
 .../dictionary/ReverseDictionaryCache.java      |   11 +-
 .../core/carbon/ColumnIdentifier.java           |  109 +
 .../core/carbon/datastore/BlockIndexStore.java  |  184 +-
 .../carbon/datastore/SegmentTaskIndexStore.java |   21 +-
 .../datastore/block/SegmentProperties.java      |   94 +-
 .../carbon/datastore/block/TableBlockInfo.java  |   18 +-
 .../impl/ColumnGroupDimensionDataChunk.java     |    7 +-
 .../ThriftWrapperSchemaConverterImpl.java       |    4 +
 .../carbon/metadata/index/BlockIndexInfo.java   |   92 +
 .../metadata/schema/table/CarbonTable.java      |    2 +-
 .../schema/table/column/CarbonColumn.java       |   23 +
 .../schema/table/column/ColumnSchema.java       |   49 +
 .../core/carbon/path/CarbonTablePath.java       |  120 +-
 .../core/constants/CarbonCommonConstants.java   |  285 +-
 .../store/columnar/ColumnGroupModel.java        |   21 -
 .../store/filesystem/HDFSCarbonFile.java        |    3 +-
 .../store/filesystem/ViewFSCarbonFile.java      |    6 +-
 .../core/datastorage/util/StoreFactory.java     |   18 +-
 .../DirectDictionaryGenerator.java              |   13 +
 .../TimeStampDirectDictionaryGenerator.java     |   46 +-
 .../core/load/LoadMetadataDetails.java          |   43 +
 .../CarbonDictionaryMetadataReaderImpl.java     |   17 +-
 .../core/reader/CarbonDictionaryReaderImpl.java |   28 +-
 .../core/reader/CarbonIndexFileReader.java      |   95 +
 .../carbondata/core/reader/ThriftReader.java    |   47 +-
 .../CarbonDictionarySortIndexReaderImpl.java    |   21 +-
 .../core/service/ColumnUniqueIdService.java     |   34 +
 .../core/service/DictionaryService.java         |   92 +
 .../carbondata/core/service/PathService.java    |   38 +
 .../core/util/CarbonMetadataUtil.java           |  114 +-
 .../carbondata/core/util/CarbonProperties.java  |  401 +--
 .../org/carbondata/core/util/CarbonUtil.java    |  245 +-
 .../core/util/DataFileFooterConverter.java      |   65 +-
 .../core/writer/CarbonDictionaryWriterImpl.java |   45 +-
 .../core/writer/CarbonIndexFileWriter.java      |   64 +
 .../core/writer/HierarchyValueWriterForCSV.java |    2 +-
 .../CarbonDictionarySortIndexWriterImpl.java    |   21 +-
 .../scan/executor/infos/KeyStructureInfo.java   |   18 -
 .../scan/executor/util/QueryUtil.java           |  105 +-
 .../scan/expression/ColumnExpression.java       |    3 +-
 .../carbondata/scan/expression/Expression.java  |    4 +-
 .../scan/expression/ExpressionResult.java       |  104 +-
 .../expression/arithmetic/AddExpression.java    |    4 +-
 .../expression/arithmetic/DivideExpression.java |    4 +-
 .../arithmetic/MultiplyExpression.java          |    4 +-
 .../arithmetic/SubstractExpression.java         |    4 +-
 .../conditional/EqualToExpression.java          |    5 +-
 .../GreaterThanEqualToExpression.java           |    4 +-
 .../conditional/GreaterThanExpression.java      |    5 +-
 .../expression/conditional/InExpression.java    |    6 +-
 .../conditional/LessThanEqualToExpression.java  |    5 +-
 .../conditional/LessThanExpression.java         |    4 +-
 .../expression/conditional/ListExpression.java  |    7 +-
 .../conditional/NotEqualsExpression.java        |    4 +-
 .../expression/conditional/NotInExpression.java |    6 +-
 .../exception/FilterIllegalMemberException.java |   98 +
 .../exception/FilterUnsupportedException.java   |    1 -
 .../scan/expression/logical/AndExpression.java  |    6 +-
 .../scan/expression/logical/NotExpression.java  |    4 +-
 .../scan/expression/logical/OrExpression.java   |    4 +-
 .../scan/filter/FilterExpressionProcessor.java  |   36 +-
 .../org/carbondata/scan/filter/FilterUtil.java  |  163 +-
 .../executer/ColGroupFilterExecuterImpl.java    |  190 --
 .../ExcludeColGroupFilterExecuterImpl.java      |  137 +
 .../executer/ExcludeFilterExecuterImpl.java     |   49 +-
 .../IncludeColGroupFilterExecuterImpl.java      |  209 ++
 .../executer/IncludeFilterExecuterImpl.java     |    8 +-
 .../executer/RowLevelFilterExecuterImpl.java    |  116 +-
 .../RowLevelRangeGrtThanFiterExecuterImpl.java  |  148 +-
 ...elRangeGrtrThanEquaToFilterExecuterImpl.java |  139 +-
 ...velRangeLessThanEqualFilterExecuterImpl.java |  187 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |  190 +-
 .../RowLevelRangeTypeExecuterFacory.java        |   13 +-
 .../filter/resolver/AndFilterResolverImpl.java  |    5 +-
 .../resolver/ConditionalFilterResolverImpl.java |   19 +-
 .../filter/resolver/FilterResolverIntf.java     |    5 +-
 .../resolver/LogicalFilterResolverImpl.java     |    3 +-
 .../resolver/RestructureFilterResolverImpl.java |    5 +-
 .../RowLevelRangeFilterResolverImpl.java        |   77 +-
 .../visitor/CustomTypeDictionaryVisitor.java    |   16 +-
 .../visitor/DictionaryColumnVisitor.java        |   11 +-
 .../visitor/NoDictionaryTypeVisitor.java        |   12 +-
 .../scan/result/iterator/RawResultIterator.java |    2 +
 .../org/carbondata/scan/util/DataTypeUtil.java  |    2 +-
 .../dictionary/AbstractDictionaryCacheTest.java |    7 +-
 .../dictionary/ForwardDictionaryCacheTest.java  |    4 +-
 .../dictionary/ReverseDictionaryCacheTest.java  |    4 +-
 .../datastore/SegmentTaskIndexStoreTest.java    |  143 -
 ...CarbonDictionarySortIndexReaderImplTest.java |    6 +-
 .../writer/CarbonDictionaryWriterImplTest.java  |    9 +-
 .../core/writer/CarbonFooterWriterTest.java     |   58 +-
 ...CarbonDictionarySortIndexWriterImplTest.java |   13 +-
 dev/molap.properties.template                   |   94 -
 dev/molap.properties_spark                      |   90 -
 .../examples/GenerateDictionaryExample.scala    |    4 +-
 format/src/main/thrift/carbondataindex.thrift   |   45 +
 format/src/main/thrift/schema.thrift            |   13 +-
 .../org/carbondata/hadoop/CarbonPathFilter.java |    4 +-
 .../AbstractDictionaryDecodedReadSupport.java   |    4 +-
 .../hadoop/test/util/StoreCreator.java          |   11 +-
 .../FACT_UNITED_DATA_INFO_sample_cube.csv       |   20 -
 .../FACT_UNITED_DATA_INFO_sample_table.csv      |   20 +
 .../src/test/resources/restructure_cube.csv     |  101 -
 .../src/test/resources/restructure_table.csv    |  101 +
 .../allqueries/AllDataTypesTestCase1.scala      |   82 +-
 .../allqueries/AllDataTypesTestCase2.scala      |  116 +-
 .../allqueries/AllDataTypesTestCase3.scala      |  211 +-
 .../allqueries/AllDataTypesTestCase4.scala      | 2576 +++++-------------
 .../allqueries/AllDataTypesTestCase5.scala      |  165 +-
 .../allqueries/AllDataTypesTestCase6.scala      | 2377 ++++------------
 .../spark/merger/RowResultMerger.java           |   19 +-
 .../carbondata/spark/load/CarbonLoadModel.java  |    2 +
 .../carbondata/spark/load/CarbonLoaderUtil.java |  189 +-
 .../spark/load/DeleteLoadFolders.java           |    5 +-
 .../spark/merger/CarbonDataMergerUtil.java      |  401 +--
 .../spark/sql/CarbonDatasourceRelation.scala    |   22 +-
 .../spark/sql/CarbonDictionaryDecoder.scala     |    7 +-
 .../org/apache/spark/sql/CarbonSqlParser.scala  |  374 +--
 .../execution/command/carbonTableSchema.scala   |  180 +-
 .../spark/sql/hive/CarbonMetastoreCatalog.scala |   85 +-
 .../spark/sql/hive/CarbonStrategies.scala       |    4 +-
 .../spark/CarbonColumnValidator.scala           |   36 +
 .../carbondata/spark/CarbonSparkFactory.scala   |   63 +
 .../spark/DictionaryDetailHelper.scala          |   66 +
 .../spark/rdd/CarbonDataLoadRDD.scala           |    4 +
 .../spark/rdd/CarbonDataRDDFactory.scala        |  240 +-
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |    7 +-
 .../carbondata/spark/rdd/CarbonScanRDD.scala    |   13 +-
 .../org/carbondata/spark/rdd/Compactor.scala    |   19 +-
 .../org/carbondata/spark/util/CommonUtil.scala  |   79 +-
 .../spark/util/GlobalDictionaryUtil.scala       |   85 +-
 .../spark/load/CarbonLoaderUtilTest.java        |    4 +-
 .../validation/FileFooterValidator.java         |    6 +-
 integration/spark/src/test/resources/data2.csv  |    6 +-
 .../src/test/resources/data2_DiffTimeFormat.csv |    4 +
 .../resources/datawithcomplexspecialchar.csv    |  151 +
 .../test/resources/datawithescapecharacter.csv  |   22 +
 .../src/test/resources/datawithnullmeasure.csv  |    5 +
 .../test/resources/datawithspecialcharacter.csv |   38 +
 .../spark/src/test/resources/filter/emp2.csv    |    9 +
 .../src/test/resources/filter/emp2allnull.csv   |    9 +
 .../src/test/resources/filter/emp2nonull.csv    |    8 +
 .../test/resources/nullandnonparsableValue.csv  |    3 +
 .../apache/spark/sql/TestCarbonSqlParser.scala  |   37 +-
 .../dataload/TestLoadDataWithBlankLine.scala    |   18 +-
 .../TestLoadDataWithEmptyArrayColumns.scala     |   12 +-
 .../TestLoadDataWithMaxMinInteger.scala         |    6 +-
 .../dataload/TestLoadDataWithNullMeasures.scala |   15 +-
 .../AllDataTypesTestCaseAggregate.scala         |   44 +-
 .../aggquery/IntegerDataTypeTestCase.scala      |    7 +-
 .../AllDataTypesTestCaseAggregate.scala         |    8 +-
 .../testsuite/bigdecimal/TestBigDecimal.scala   |   10 +
 .../bigdecimal/TestNullAndEmptyFields.scala     |  110 +
 .../createtable/TestCreateTableSyntax.scala     |   32 +-
 .../DataCompactionCardinalityBoundryTest.scala  |   12 +-
 .../datacompaction/DataCompactionLockTest.scala |  111 +
 .../datacompaction/DataCompactionTest.scala     |   12 +-
 .../dataload/TestLoadDataWithHiveSyntax.scala   |  108 +-
 ...ataWithMalformedCarbonCommandException.scala |    4 +-
 .../dataretention/DataRetentionTestCase.scala   |   37 +
 .../detailquery/AllDataTypesTestCase.scala      |    6 +-
 .../ColumnGroupDataTypesTestCase.scala          |   51 +-
 .../ColumnPropertyValidationTestCase.scala      |   28 +
 .../HighCardinalityDataTypesTestCase.scala      |   66 +-
 .../detailquery/IntegerDataTypeTestCase.scala   |    6 +-
 ...estampDataTypeDirectDictionaryTestCase.scala |   12 +-
 ...TypeDirectDictionaryWithNoDictTestCase.scala |    4 +-
 .../filterexpr/AllDataTypesTestCaseFilter.scala |    6 +-
 .../filterexpr/FilterProcessorTestCase.scala    |  131 +-
 .../GrtLtFilterProcessorTestCase.scala          |  176 ++
 .../filterexpr/IntegerDataTypeTestCase.scala    |    6 +-
 .../NullMeasureValueTestCaseFilter.scala        |   39 +
 .../HadoopFSRelationTestCase.scala              |   16 +-
 .../joinquery/AllDataTypesTestCaseJoin.scala    |    6 +-
 .../joinquery/IntegerDataTypeTestCase.scala     |    6 +-
 .../NullMeasureValueTestCaseAggregate.scala     |    2 +-
 .../sortexpr/AllDataTypesTestCaseSort.scala     |    6 +-
 .../sortexpr/IntegerDataTypeTestCase.scala      |   13 +-
 .../util/GlobalDictionaryUtilTestCase.scala     |   51 +-
 .../carbonaggregatesurrogategenerator/icon.png  |  Bin 9058 -> 0 bytes
 .../plugin.xml                                  |    1 -
 .../steps/carbonautoagggraphgenerator/icon.png  |  Bin 9058 -> 0 bytes
 .../carbonautoagggraphgenerator/plugin.xml      |    1 -
 .../steps/carbonautoaggslicemerger/icon.png     |  Bin 9058 -> 0 bytes
 .../steps/carbonautoaggslicemerger/plugin.xml   |    1 -
 .../plugins/steps/carboncsvbasedseqgen/icon.png |  Bin 24093 -> 0 bytes
 .../steps/carboncsvbasedseqgen/plugin.xml       |    1 -
 .../plugins/steps/carboncsvreader/icon.png      |  Bin 24093 -> 0 bytes
 .../plugins/steps/carboncsvreader/plugin.xml    |    1 -
 .../plugins/steps/carboncsvreaderstrep/icon.png |  Bin 9058 -> 0 bytes
 .../steps/carboncsvreaderstrep/plugin.xml       |    1 -
 .../steps/carbondatawriter/Binary-icon.png      |  Bin 14573 -> 0 bytes
 .../plugins/steps/carbondatawriter/plugin.xml   |    1 -
 .../plugins/steps/carbonfactreader/icon.png     |  Bin 9058 -> 0 bytes
 .../plugins/steps/carbonfactreader/plugin.xml   |    1 -
 .../plugins/steps/carbongroupby/icon.png        |  Bin 9058 -> 0 bytes
 .../plugins/steps/carbongroupby/plugin.xml      |    1 -
 .../steps/carboninmemoryfactreader/icon.png     |  Bin 9058 -> 0 bytes
 .../steps/carboninmemoryfactreader/plugin.xml   |    1 -
 .../.kettle/plugins/steps/carbonseqgen/icon.png |  Bin 24093 -> 0 bytes
 .../plugins/steps/carbonseqgen/plugin.xml       |    1 -
 .../plugins/steps/carbonslicemerger/icon.png    |  Bin 9058 -> 0 bytes
 .../plugins/steps/carbonslicemerger/plugin.xml  |    1 -
 .../steps/carbonsortkeyandgroupby/icon.png      |  Bin 9058 -> 0 bytes
 .../steps/carbonsortkeyandgroupby/plugin.xml    |    1 -
 .../.kettle/plugins/steps/mdkeygenstep/icon.png |  Bin 9058 -> 0 bytes
 .../plugins/steps/mdkeygenstep/plugin.xml       |    1 -
 .../.kettle/plugins/steps/sortkeystep/icon.png  |  Bin 9058 -> 0 bytes
 .../plugins/steps/sortkeystep/plugin.xml        |    1 -
 .../org/carbondata/lcm/locks/LocalFileLock.java |    4 +-
 .../org/carbondata/lcm/locks/LockUsage.java     |    3 +-
 .../lcm/status/SegmentStatusManager.java        |  111 +-
 .../api/dataloader/DataLoadModel.java           |    9 +
 .../processing/csvload/DataGraphExecuter.java   |    8 +
 .../csvreaderstep/BlockDataHandler.java         |   83 +-
 .../processing/csvreaderstep/CsvInput.java      |    4 +-
 .../processing/csvreaderstep/CsvInputData.java  |    2 +
 .../processing/csvreaderstep/CsvInputMeta.java  |   21 +-
 .../dataprocessor/DataProcessTaskStatus.java    |   10 +
 .../dataprocessor/IDataProcessStatus.java       |    2 +
 .../graphgenerator/GraphGenerator.java          |    6 +
 .../configuration/GraphConfigurationInfo.java   |   10 +
 .../processing/mdkeygen/MDKeyGenStep.java       |   40 +-
 .../merger/step/CarbonSliceMergerStep.java      |    3 +-
 .../processing/schema/metadata/ColumnsInfo.java |   10 +
 .../store/CarbonFactDataHandlerColumnar.java    |  111 +-
 .../store/CarbonFactDataHandlerModel.java       |   63 +-
 .../store/SingleThreadFinalSortFilesMerger.java |    6 +-
 .../store/colgroup/ColGroupDataHolder.java      |    4 +-
 .../store/colgroup/ColGroupMinMax.java          |   30 +-
 .../store/writer/AbstractFactDataWriter.java    |  147 +-
 ...actDataWriterImplForIntIndexAndAggBlock.java |    6 +-
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |   36 +
 .../csvbased/CarbonCSVBasedSeqGenStep.java      |   19 +-
 .../FileStoreSurrogateKeyGenForCSV.java         |   12 +-
 .../processing/util/CarbonSchemaParser.java     |   44 +
 .../store/colgroup/ColGroupMinMaxTest.java      |  385 +--
 250 files changed, 8562 insertions(+), 7434 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
index c177feb,f6f369a..bfa5cdc
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
@@@ -32,6 -34,6 +34,7 @@@ import org.carbondata.core.carbon.metad
  import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
  import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
  import org.carbondata.core.constants.CarbonCommonConstants;
++import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
  import org.carbondata.core.keygenerator.KeyGenerator;
  import org.carbondata.core.keygenerator.columnar.ColumnarSplitter;
  import org.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
index 532c6e9,0000000..26b2519
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
@@@ -1,137 -1,0 +1,116 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *    http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.carbondata.core.datastorage.store.columnar;
 +
 +public class ColumnGroupModel {
 +
 +  /**
-    * cardinality all dimension
-    */
-   private int[] colGroupCardinality;
- 
-   /**
 +   * number of columns in columnar block
 +   */
 +  private int[] columnSplit;
 +
 +  /**
 +   * total number of columns
 +   */
 +  private int noOfColumnsStore;
 +
 +  /**
 +   * whether given index is columnar or not
 +   * true: columnar
 +   * false: row block
 +   */
 +  private boolean[] columnarStore;
 +
 +  /**
 +   * column groups
 +   * e.g
 +   * {{0,1,2},3,4,{5,6}}
 +   */
 +  private int[][] columnGroups;
 +
 +  /**
-    * @return cardinality of column groups
-    */
-   public int[] getColumnGroupCardinality() {
-     return colGroupCardinality;
-   }
- 
-   /**
-    * set columngroup cardinality
-    *
-    * @param columnGroupCardinality
-    */
-   public void setColumnGroupCardinality(int[] columnGroupCardinality) {
-     this.colGroupCardinality = columnGroupCardinality;
-   }
- 
-   /**
 +   * return columnSplit
 +   *
 +   * @return
 +   */
 +  public int[] getColumnSplit() {
 +    return columnSplit;
 +  }
 +
 +  /**
 +   * set columnSplit
 +   *
 +   * @param split
 +   */
 +  public void setColumnSplit(int[] split) {
 +    this.columnSplit = split;
 +  }
 +
 +  /**
 +   * @return no of columnar block
 +   */
 +  public int getNoOfColumnStore() {
 +    return this.noOfColumnsStore;
 +  }
 +
 +  /**
 +   * set no of columnar block
 +   *
 +   * @param noOfColumnsStore
 +   */
 +  public void setNoOfColumnStore(int noOfColumnsStore) {
 +    this.noOfColumnsStore = noOfColumnsStore;
 +  }
 +
 +  /**
 +   * it's an identifier for row block or single column block
 +   *
 +   * @param columnarStore
 +   */
 +  public void setColumnarStore(boolean[] columnarStore) {
 +    this.columnarStore = columnarStore;
 +  }
 +
 +  /**
 +   * set column groups
 +   *
 +   * @param columnGroups
 +   */
 +  public void setColumnGroup(int[][] columnGroups) {
 +    this.columnGroups = columnGroups;
 +  }
 +
 +  /**
 +   * check if given column group is columnar
 +   *
 +   * @param colGroup
 +   * @return true if given block is columnar
 +   */
 +  public boolean isColumnar(int colGroup) {
 +    return columnarStore[colGroup];
 +  }
 +
 +  /**
 +   * @return columngroups
 +   */
 +  public int[][] getColumnGroup() {
 +    return this.columnGroups;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
index 8fc5aac,63f8157..e8efa21
--- a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
@@@ -19,35 -19,38 +19,19 @@@
  
  package org.carbondata.core.datastorage.util;
  
- import org.carbondata.core.constants.CarbonCommonConstants;
 -import org.carbondata.core.datastorage.store.FileHolder;
 -import org.carbondata.core.datastorage.store.NodeKeyStore;
  import org.carbondata.core.datastorage.store.NodeMeasureDataStore;
 -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStore;
 -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
  import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
 -import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataFileStore;
  import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
- 
 -import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataFileStore;
  import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore;
- import org.carbondata.core.util.CarbonProperties;
 -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarFileKeyStore;
 -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarInMemoryStore;
 -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarFileKeyStore;
 -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarInMemoryStore;
 -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyFileStore;
 -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyInMemoryStore;
 -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyFileStore;
 -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyInMemoryStore;
  
  public final class StoreFactory {
    /**
-    * Double array data store.
 -   * key type.
--   */
-   private static final String COMPRESSED_DOUBLE_ARRAY = "COMPRESSED_DOUBLE_ARRAY";
 -  private static StoreType keyType;
--  /**
     * value type.
     */
    private static StoreType valueType;
  
    static {
-     String valuetype = CarbonProperties.getInstance()
-         .getProperty(CarbonCommonConstants.VALUESTORE_TYPE,
-             CarbonCommonConstants.VALUESTORE_TYPE_DEFAULT_VAL);
- 
-     // set value type
-     if (COMPRESSED_DOUBLE_ARRAY.equals(valuetype)) {
-       valueType = StoreType.COMPRESSED_DOUBLE_ARRAY;
-     } else {
-       valueType = StoreType.HEAVY_VALUE_COMPRESSION;
-     }
 -    keyType = StoreType.COMPRESSED_SINGLE_ARRAY;
+     valueType = StoreType.HEAVY_VALUE_COMPRESSION;
    }
  
    private StoreFactory() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 771e5f5,97baf3d..0320621
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@@ -39,20 -39,7 +39,14 @@@ import static org.carbondata.core.keyge
   */
  public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator {
  
 +  private TimeStampDirectDictionaryGenerator() {
 +
 +  }
 +
 +  public static TimeStampDirectDictionaryGenerator instance =
 +      new TimeStampDirectDictionaryGenerator();
 +
    /**
-    * Logger instance
-    */
-   private static final LogService LOGGER =
-       LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
- 
-   /**
     * The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis.
     */
    public static final long granularityFactor;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/util/CarbonUtil.java
index 31d41b3,b19f8d3..2368139
--- a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
@@@ -47,9 -40,14 +40,13 @@@ import org.carbondata.core.carbon.metad
  import org.carbondata.core.carbon.metadata.datatype.DataType;
  import org.carbondata.core.carbon.metadata.encoder.Encoding;
  import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+ import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
+ import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+ import org.carbondata.core.carbon.path.CarbonStorePath;
+ import org.carbondata.core.carbon.path.CarbonTablePath;
  import org.carbondata.core.constants.CarbonCommonConstants;
 -import org.carbondata.core.datastorage.store.FileHolder;
 +import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
  import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
 -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
  import org.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
  import org.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
  import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
index 6641c18,0000000..52a139c
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
@@@ -1,418 -1,0 +1,463 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *    http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.carbondata.core.util;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.IOException;
 +import java.io.ObjectInputStream;
 +import java.nio.ByteBuffer;
 +import java.util.ArrayList;
 +import java.util.BitSet;
 +import java.util.Iterator;
 +import java.util.List;
 +
 +import org.carbondata.common.logging.LogService;
 +import org.carbondata.common.logging.LogServiceFactory;
++import org.carbondata.core.carbon.datastore.block.TableBlockInfo;
 +import org.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
 +import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
 +import org.carbondata.core.carbon.metadata.blocklet.SegmentInfo;
 +import org.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta;
 +import org.carbondata.core.carbon.metadata.blocklet.compressor.CompressionCodec;
 +import org.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
 +import org.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
 +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletBTreeIndex;
 +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
 +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex;
 +import org.carbondata.core.carbon.metadata.blocklet.sort.SortState;
 +import org.carbondata.core.carbon.metadata.datatype.DataType;
 +import org.carbondata.core.carbon.metadata.encoder.Encoding;
 +import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
 +import org.carbondata.core.datastorage.store.FileHolder;
 +import org.carbondata.core.datastorage.store.impl.FileFactory;
 +import org.carbondata.core.metadata.ValueEncoderMeta;
 +import org.carbondata.core.reader.CarbonFooterReader;
++import org.carbondata.core.reader.CarbonIndexFileReader;
++import org.carbondata.format.BlockIndex;
 +import org.carbondata.format.FileFooter;
 +
 +/**
 + * Below class will be used to convert the thrift object of data file
 + * meta data to wrapper object
 + */
- class DataFileFooterConverter {
++public class DataFileFooterConverter {
 +
 +  private static final LogService LOGGER =
 +      LogServiceFactory.getLogService(DataFileFooterConverter.class.getName());
 +
 +  /**
++   * Below method will be used to get the index info from index file
++   *
++   * @param filePath           file path of the index file
++   * @param tableBlockInfoList table block index
++   * @return list of index info
++   * @throws IOException problem while reading the index file
++   */
++  public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList)
++      throws IOException {
++    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
++    // open the reader
++    indexReader.openThriftReader(filePath);
++    // get the index header
++    org.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
++    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
++    List<org.carbondata.format.ColumnSchema> table_columns = readIndexHeader.getTable_columns();
++    for (int i = 0; i < table_columns.size(); i++) {
++      columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
++    }
++    // get the segment info
++    SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
++    BlockletIndex blockletIndex = null;
++    int counter = 0;
++    DataFileFooter dataFileFooter = null;
++    List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
++    // read the block info from file
++    while (indexReader.hasNext()) {
++      BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
++      blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
++      dataFileFooter = new DataFileFooter();
++      dataFileFooter.setBlockletIndex(blockletIndex);
++      dataFileFooter.setColumnInTable(columnSchemaList);
++      dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
++      dataFileFooter.setTableBlockInfo(tableBlockInfoList.get(counter++));
++      dataFileFooter.setSegmentInfo(segmentInfo);
++      dataFileFooters.add(dataFileFooter);
++    }
++    return dataFileFooters;
++  }
++
++  /**
 +   * Below method will be used to convert thrift file meta to wrapper file meta
 +   */
 +  public DataFileFooter readDataFileFooter(String filePath, long blockOffset, long blockLength)
 +      throws IOException {
 +    DataFileFooter dataFileFooter = new DataFileFooter();
 +    FileHolder fileReader = null;
 +    try {
 +      long completeBlockLength = blockOffset + blockLength;
 +      long footerPointer = completeBlockLength - 8;
 +      fileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath));
 +      long actualFooterOffset = fileReader.readLong(filePath, footerPointer);
 +      CarbonFooterReader reader = new CarbonFooterReader(filePath, actualFooterOffset);
 +      FileFooter footer = reader.readFooter();
 +      dataFileFooter.setVersionId(footer.getVersion());
 +      dataFileFooter.setNumberOfRows(footer.getNum_rows());
 +      dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
 +      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
 +      List<org.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
 +      for (int i = 0; i < table_columns.size(); i++) {
 +        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
 +      }
 +      dataFileFooter.setColumnInTable(columnSchemaList);
 +
 +      List<org.carbondata.format.BlockletIndex> leaf_node_indices_Thrift =
 +          footer.getBlocklet_index_list();
 +      List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
 +      for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
 +        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
 +        blockletIndexList.add(blockletIndex);
 +      }
 +
 +      List<org.carbondata.format.BlockletInfo> leaf_node_infos_Thrift =
 +          footer.getBlocklet_info_list();
 +      List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
 +      for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
 +        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
 +        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
 +        blockletInfoList.add(blockletInfo);
 +      }
 +      dataFileFooter.setBlockletList(blockletInfoList);
 +      dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
 +    } finally {
 +      if (null != fileReader) {
 +        fileReader.finish();
 +      }
 +    }
 +    return dataFileFooter;
 +  }
 +
 +  /**
 +   * Below method will be used to get blocklet index for data file meta
 +   *
 +   * @param blockletIndexList
 +   * @return blocklet index
 +   */
 +  private BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
 +    BlockletIndex blockletIndex = new BlockletIndex();
 +    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
 +    blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
 +    blockletBTreeIndex
 +        .setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
 +    blockletIndex.setBtreeIndex(blockletBTreeIndex);
 +    byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
 +    byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
 +    byte[][] minValue = null;
 +    byte[][] maxValue = null;
 +    for (int i = 1; i < blockletIndexList.size(); i++) {
 +      minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
 +      maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
 +      for (int j = 0; j < maxValue.length; j++) {
 +        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
 +          currentMinValue[j] = minValue[j].clone();
 +        }
 +        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
 +          currentMaxValue[j] = maxValue[j].clone();
 +        }
 +      }
 +    }
 +
 +    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
 +    minMax.setMaxValues(currentMaxValue);
 +    minMax.setMinValues(currentMinValue);
 +    blockletIndex.setMinMaxIndex(minMax);
 +    return blockletIndex;
 +  }
 +
 +  private ColumnSchema thriftColumnSchmeaToWrapperColumnSchema(
 +      org.carbondata.format.ColumnSchema externalColumnSchema) {
 +    ColumnSchema wrapperColumnSchema = new ColumnSchema();
 +    wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
 +    wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
 +    wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
 +    wrapperColumnSchema
 +        .setDataType(thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type));
 +    wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
 +    List<Encoding> encoders = new ArrayList<Encoding>();
 +    for (org.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
 +      encoders.add(fromExternalToWrapperEncoding(encoder));
 +    }
 +    wrapperColumnSchema.setEncodingList(encoders);
 +    wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
 +    wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
 +    wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
 +    wrapperColumnSchema.setScale(externalColumnSchema.getScale());
 +    wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
 +    wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function());
 +    return wrapperColumnSchema;
 +  }
 +
 +  /**
 +   * Below method is to convert the blocklet info of the thrift to wrapper
 +   * blocklet info
 +   *
 +   * @param blockletInfoThrift blocklet info of the thrift
 +   * @return blocklet info wrapper
 +   */
 +  private BlockletInfo getBlockletInfo(org.carbondata.format.BlockletInfo blockletInfoThrift) {
 +    BlockletInfo blockletInfo = new BlockletInfo();
 +    List<DataChunk> dimensionColumnChunk = new ArrayList<DataChunk>();
 +    List<DataChunk> measureChunk = new ArrayList<DataChunk>();
 +    Iterator<org.carbondata.format.DataChunk> column_data_chunksIterator =
 +        blockletInfoThrift.getColumn_data_chunksIterator();
-     while (column_data_chunksIterator.hasNext()) {
-       org.carbondata.format.DataChunk next = column_data_chunksIterator.next();
-       if (next.isRowMajor()) {
-         dimensionColumnChunk.add(getDataChunk(next, false));
-       } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) {
-         measureChunk.add(getDataChunk(next, true));
-       } else {
- 
-         dimensionColumnChunk.add(getDataChunk(next, false));
++    if (null != column_data_chunksIterator) {
++      while (column_data_chunksIterator.hasNext()) {
++        org.carbondata.format.DataChunk next = column_data_chunksIterator.next();
++        if (next.isRowMajor()) {
++          dimensionColumnChunk.add(getDataChunk(next, false));
++        } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) {
++          measureChunk.add(getDataChunk(next, true));
++        } else {
++          dimensionColumnChunk.add(getDataChunk(next, false));
++        }
 +      }
 +    }
 +    blockletInfo.setDimensionColumnChunk(dimensionColumnChunk);
 +    blockletInfo.setMeasureColumnChunk(measureChunk);
 +    blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
 +    return blockletInfo;
 +  }
 +
 +  /**
 +   * Below method is convert the thrift encoding to wrapper encoding
 +   *
 +   * @param encoderThrift thrift encoding
 +   * @return wrapper encoding
 +   */
 +  private Encoding fromExternalToWrapperEncoding(org.carbondata.format.Encoding encoderThrift) {
 +    switch (encoderThrift) {
 +      case DICTIONARY:
 +        return Encoding.DICTIONARY;
 +      case DELTA:
 +        return Encoding.DELTA;
 +      case RLE:
 +        return Encoding.RLE;
 +      case INVERTED_INDEX:
 +        return Encoding.INVERTED_INDEX;
 +      case BIT_PACKED:
 +        return Encoding.BIT_PACKED;
 +      case DIRECT_DICTIONARY:
 +        return Encoding.DIRECT_DICTIONARY;
 +      default:
 +        return Encoding.DICTIONARY;
 +    }
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift compression to wrapper
 +   * compression codec
 +   *
 +   * @param compressionCodecThrift
 +   * @return wrapper compression codec
 +   */
 +  private CompressionCodec getCompressionCodec(
 +      org.carbondata.format.CompressionCodec compressionCodecThrift) {
 +    switch (compressionCodecThrift) {
 +      case SNAPPY:
 +        return CompressionCodec.SNAPPY;
 +      default:
 +        return CompressionCodec.SNAPPY;
 +    }
 +  }
 +
 +  /**
 +   * Below method will be used to convert thrift segment object to wrapper
 +   * segment object
 +   *
 +   * @param segmentInfo thrift segment info object
 +   * @return wrapper segment info object
 +   */
 +  private SegmentInfo getSegmentInfo(org.carbondata.format.SegmentInfo segmentInfo) {
 +    SegmentInfo info = new SegmentInfo();
 +    int[] cardinality = new int[segmentInfo.getColumn_cardinalities().size()];
 +    for (int i = 0; i < cardinality.length; i++) {
 +      cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
 +    }
 +    info.setColumnCardinality(cardinality);
 +    info.setNumberOfColumns(segmentInfo.getNum_cols());
 +    return info;
 +  }
 +
 +  /**
 +   * Below method will be used to convert the blocklet index of thrift to
 +   * wrapper
 +   *
 +   * @param blockletIndexThrift
 +   * @return blocklet index wrapper
 +   */
 +  private BlockletIndex getBlockletIndex(org.carbondata.format.BlockletIndex blockletIndexThrift) {
 +    org.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index();
 +    org.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index();
 +    return new BlockletIndex(
 +        new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()),
 +        new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift compression meta to
 +   * wrapper chunk compression meta
 +   *
 +   * @param chunkCompressionMetaThrift
 +   * @return chunkCompressionMetaWrapper
 +   */
 +  private ChunkCompressorMeta getChunkCompressionMeta(
 +      org.carbondata.format.ChunkCompressionMeta chunkCompressionMetaThrift) {
 +    ChunkCompressorMeta compressorMeta = new ChunkCompressorMeta();
 +    compressorMeta
 +        .setCompressor(getCompressionCodec(chunkCompressionMetaThrift.getCompression_codec()));
 +    compressorMeta.setCompressedSize(chunkCompressionMetaThrift.getTotal_compressed_size());
 +    compressorMeta.setUncompressedSize(chunkCompressionMetaThrift.getTotal_uncompressed_size());
 +    return compressorMeta;
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift data type to wrapper data
 +   * type
 +   *
 +   * @param dataTypeThrift
 +   * @return dataType wrapper
 +   */
 +  private DataType thriftDataTyopeToWrapperDataType(org.carbondata.format.DataType dataTypeThrift) {
 +    switch (dataTypeThrift) {
 +      case STRING:
 +        return DataType.STRING;
 +      case SHORT:
 +        return DataType.SHORT;
 +      case INT:
 +        return DataType.INT;
 +      case LONG:
 +        return DataType.LONG;
 +      case DOUBLE:
 +        return DataType.DOUBLE;
 +      case DECIMAL:
 +        return DataType.DECIMAL;
 +      case TIMESTAMP:
 +        return DataType.TIMESTAMP;
 +      case ARRAY:
 +        return DataType.ARRAY;
 +      case STRUCT:
 +        return DataType.STRUCT;
 +      default:
 +        return DataType.STRING;
 +    }
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift presence meta to wrapper
 +   * presence meta
 +   *
 +   * @param presentMetadataThrift
 +   * @return wrapper presence meta
 +   */
 +  private PresenceMeta getPresenceMeta(org.carbondata.format.PresenceMeta presentMetadataThrift) {
 +    PresenceMeta presenceMeta = new PresenceMeta();
 +    presenceMeta.setRepresentNullValues(presentMetadataThrift.isRepresents_presence());
 +    presenceMeta.setBitSet(BitSet.valueOf(presentMetadataThrift.getPresent_bit_stream()));
 +    return presenceMeta;
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift object to wrapper object
 +   *
 +   * @param sortStateThrift
 +   * @return wrapper sort state object
 +   */
 +  private SortState getSortState(org.carbondata.format.SortState sortStateThrift) {
 +    if (sortStateThrift == org.carbondata.format.SortState.SORT_EXPLICIT) {
 +      return SortState.SORT_EXPLICT;
 +    } else if (sortStateThrift == org.carbondata.format.SortState.SORT_NATIVE) {
 +      return SortState.SORT_NATIVE;
 +    } else {
 +      return SortState.SORT_NONE;
 +    }
 +  }
 +
 +  /**
 +   * Below method will be used to convert the thrift data chunk to wrapper
 +   * data chunk
 +   *
 +   * @param datachunkThrift
 +   * @return wrapper data chunk
 +   */
 +  private DataChunk getDataChunk(org.carbondata.format.DataChunk datachunkThrift,
 +      boolean isPresenceMetaPresent) {
 +    DataChunk dataChunk = new DataChunk();
 +    dataChunk.setColumnUniqueIdList(datachunkThrift.getColumn_ids());
 +    dataChunk.setDataPageLength(datachunkThrift.getData_page_length());
 +    dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());
 +    if (isPresenceMetaPresent) {
 +      dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence()));
 +    }
 +    dataChunk.setRlePageLength(datachunkThrift.getRle_page_length());
 +    dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset());
 +    dataChunk.setRowMajor(datachunkThrift.isRowMajor());
 +    dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());
 +    dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());
 +    dataChunk.setSortState(getSortState(datachunkThrift.getSort_state()));
 +    dataChunk.setChunkCompressionMeta(getChunkCompressionMeta(datachunkThrift.getChunk_meta()));
 +    List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());
 +    for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {
 +      encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
 +    }
 +    dataChunk.setEncoderList(encodingList);
 +    if (encodingList.contains(Encoding.DELTA)) {
 +      List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();
 +      List<ValueEncoderMeta> encodeMetaList =
 +          new ArrayList<ValueEncoderMeta>(thriftEncoderMeta.size());
 +      for (int i = 0; i < thriftEncoderMeta.size(); i++) {
 +        encodeMetaList.add(deserializeEncoderMeta(thriftEncoderMeta.get(i).array()));
 +      }
 +      dataChunk.setValueEncoderMeta(encodeMetaList);
 +    }
 +    return dataChunk;
 +  }
 +
 +  /**
 +   * Below method will be used to convert the encode metadata to
 +   * ValueEncoderMeta object
 +   *
 +   * @param encoderMeta
 +   * @return ValueEncoderMeta object
 +   */
 +  private ValueEncoderMeta deserializeEncoderMeta(byte[] encoderMeta) {
 +    // TODO : should remove the unnecessary fields.
 +    ByteArrayInputStream aos = null;
 +    ObjectInputStream objStream = null;
 +    ValueEncoderMeta meta = null;
 +    try {
 +      aos = new ByteArrayInputStream(encoderMeta);
 +      objStream = new ObjectInputStream(aos);
 +      meta = (ValueEncoderMeta) objStream.readObject();
 +    } catch (ClassNotFoundException e) {
 +      LOGGER.error(e);
 +    } catch (IOException e) {
 +      CarbonUtil.closeStreams(objStream);
 +    }
 +    return meta;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
index 51874b4,0000000..e19a3e9
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
@@@ -1,125 -1,0 +1,107 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *    http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.carbondata.scan.executor.infos;
 +
 +import org.carbondata.core.keygenerator.KeyGenerator;
 +
 +/**
 + * Below class will store the structure of the key
 + * used during query execution
 + */
 +public class KeyStructureInfo {
 +
 +  /**
 +   * it's actually a latest key generator
 +   * last table block as this key generator will be used to
 +   * to update the mdkey of the older slice with the new slice
 +   */
 +  private KeyGenerator keyGenerator;
 +
 +  /**
 +   * mask bytes ranges for the query
 +   */
 +  private int[] maskByteRanges;
 +
 +  /**
 +   * masked bytes of the query
 +   */
 +  private int[] maskedBytes;
 +
 +  /**
 +   * max key for query execution
 +   */
 +  private byte[] maxKey;
 +
 +  /**
-    * mdkey start index of block
-    */
-   private int blockMdKeyStartOffset;
- 
-   /**
 +   * @return the keyGenerator
 +   */
 +  public KeyGenerator getKeyGenerator() {
 +    return keyGenerator;
 +  }
 +
 +  /**
 +   * @param keyGenerator the keyGenerator to set
 +   */
 +  public void setKeyGenerator(KeyGenerator keyGenerator) {
 +    this.keyGenerator = keyGenerator;
 +  }
 +
 +  /**
 +   * @return the maskByteRanges
 +   */
 +  public int[] getMaskByteRanges() {
 +    return maskByteRanges;
 +  }
 +
 +  /**
 +   * @param maskByteRanges the maskByteRanges to set
 +   */
 +  public void setMaskByteRanges(int[] maskByteRanges) {
 +    this.maskByteRanges = maskByteRanges;
 +  }
 +
 +  /**
 +   * @return the maskedBytes
 +   */
 +  public int[] getMaskedBytes() {
 +    return maskedBytes;
 +  }
 +
 +  /**
 +   * @param maskedBytes the maskedBytes to set
 +   */
 +  public void setMaskedBytes(int[] maskedBytes) {
 +    this.maskedBytes = maskedBytes;
 +  }
 +
 +  /**
 +   * @return the maxKey
 +   */
 +  public byte[] getMaxKey() {
 +    return maxKey;
 +  }
 +
 +  /**
 +   * @param maxKey the maxKey to set
 +   */
 +  public void setMaxKey(byte[] maxKey) {
 +    this.maxKey = maxKey;
 +  }
 +
-   /**
-    * @param startOffset
-    */
-   public void setBlockMdKeyStartOffset(int startOffset) {
-     this.blockMdKeyStartOffset = startOffset;
-   }
- 
-   /**
-    * @return
-    */
-   public int getBlockMdKeyStartOffset() {
-     return this.blockMdKeyStartOffset;
-   }
 +}


Mime
View raw message