Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C06E7200B29 for ; Thu, 30 Jun 2016 19:42:03 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BF0D6160A72; Thu, 30 Jun 2016 17:42:03 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C1200160A6B for ; Thu, 30 Jun 2016 19:42:01 +0200 (CEST) Received: (qmail 47886 invoked by uid 500); 30 Jun 2016 17:42:01 -0000 Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.incubator.apache.org Delivered-To: mailing list commits@carbondata.incubator.apache.org Received: (qmail 47877 invoked by uid 99); 30 Jun 2016 17:42:00 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Jun 2016 17:42:00 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 79C1D1A01B9 for ; Thu, 30 Jun 2016 17:42:00 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.646 X-Spam-Level: X-Spam-Status: No, score=-4.646 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.426] autolearn=disabled Received: from mx2-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id FtxiNx3CJ1Dd for ; Thu, 30 Jun 2016 17:41:52 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-eu.apache.org (ASF Mail Server at mx2-lw-eu.apache.org) with SMTP id 79A7060D13 for ; Thu, 30 Jun 2016 17:41:50 +0000 (UTC) Received: (qmail 46600 invoked by uid 99); 30 Jun 2016 17:41:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Jun 2016 17:41:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 74432E00DB; Thu, 30 Jun 2016 17:41:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chenliang613@apache.org To: commits@carbondata.incubator.apache.org Date: Thu, 30 Jun 2016 17:42:35 -0000 Message-Id: <79ded99ea1204629befe6cca7c8e0625@git.apache.org> In-Reply-To: <686ca27020db4633bf5b1c883df9389f@git.apache.org> References: <686ca27020db4633bf5b1c883df9389f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [48/50] [abbrv] incubator-carbondata git commit: Merge remote-tracking branch 'carbon_master/master' into apache/master archived-at: Thu, 30 Jun 2016 17:42:03 -0000 Merge remote-tracking branch 'carbon_master/master' into apache/master Conflicts: README.md core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java core/src/main/java/org/carbondata/core/util/CarbonUtil.java core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java core/src/main/java/org/carbondata/query/aggregator/impl/CustomAggregatorHelper.java core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java core/src/main/java/org/carbondata/query/carbon/processor/impl/AggregateQueryBlockProcessor.java core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java core/src/main/java/org/carbondata/scan/expression/Expression.java core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/ListExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java core/src/main/java/org/carbondata/scan/expression/exception/FilterUnsupportedException.java core/src/main/java/org/carbondata/scan/expression/logical/AndExpression.java core/src/main/java/org/carbondata/scan/expression/logical/NotExpression.java core/src/main/java/org/carbondata/scan/expression/logical/OrExpression.java core/src/main/java/org/carbondata/scan/filter/FilterUtil.java core/src/main/java/org/carbondata/scan/filter/executer/ColGroupFilterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java core/src/main/java/org/carbondata/scan/filter/resolver/FilterResolverIntf.java core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java core/src/main/java/org/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java core/src/main/java/org/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase1.scala integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase2.scala integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala processing/src/main/java/org/carbondata/processing/store/CarbonFactDataHandlerModel.java processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupDataHolder.java processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupMinMax.java processing/src/main/java/org/carbondata/processing/store/writer/AbstractFactDataWriter.java processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f722186 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f722186 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f722186 Branch: refs/heads/master Commit: 7f72218602926ca5f6629825df7b8f70ab1c6aa6 Parents: e9bbf75 d44c9e7 Author: ravipesala Authored: Thu Jun 30 17:41:47 2016 +0530 Committer: ravipesala Committed: Thu Jun 30 17:41:47 2016 +0530 ---------------------------------------------------------------------- common/pom.xml | 8 +- .../common/logging/impl/StandardLogService.java | 15 +- conf/carbon.properties.template | 98 + .../common/ext/ColumnUniqueIdGenerator.java | 41 + .../common/ext/DictionaryFactory.java | 119 + .../org/carbondata/common/ext/PathFactory.java | 48 + .../common/factory/CarbonCommonFactory.java | 54 + .../dictionary/AbstractDictionaryCache.java | 43 +- .../cache/dictionary/ColumnDictionaryInfo.java | 5 +- .../cache/dictionary/DictionaryCacheLoader.java | 7 +- .../dictionary/DictionaryCacheLoaderImpl.java | 28 +- .../DictionaryColumnUniqueIdentifier.java | 9 +- .../dictionary/ForwardDictionaryCache.java | 11 +- .../dictionary/ReverseDictionaryCache.java | 11 +- .../core/carbon/ColumnIdentifier.java | 109 + .../core/carbon/datastore/BlockIndexStore.java | 184 +- .../carbon/datastore/SegmentTaskIndexStore.java | 21 +- .../datastore/block/SegmentProperties.java | 94 +- .../carbon/datastore/block/TableBlockInfo.java | 18 +- .../impl/ColumnGroupDimensionDataChunk.java | 7 +- .../ThriftWrapperSchemaConverterImpl.java | 4 + .../carbon/metadata/index/BlockIndexInfo.java | 92 + .../metadata/schema/table/CarbonTable.java | 2 +- .../schema/table/column/CarbonColumn.java | 23 + .../schema/table/column/ColumnSchema.java | 49 + .../core/carbon/path/CarbonTablePath.java | 120 +- .../core/constants/CarbonCommonConstants.java | 285 +- .../store/columnar/ColumnGroupModel.java | 21 - .../store/filesystem/HDFSCarbonFile.java | 3 +- .../store/filesystem/ViewFSCarbonFile.java | 6 +- .../core/datastorage/util/StoreFactory.java | 18 +- .../DirectDictionaryGenerator.java | 13 + .../TimeStampDirectDictionaryGenerator.java | 46 +- .../core/load/LoadMetadataDetails.java | 43 + .../CarbonDictionaryMetadataReaderImpl.java | 17 +- .../core/reader/CarbonDictionaryReaderImpl.java | 28 +- .../core/reader/CarbonIndexFileReader.java | 95 + .../carbondata/core/reader/ThriftReader.java | 47 +- .../CarbonDictionarySortIndexReaderImpl.java | 21 +- .../core/service/ColumnUniqueIdService.java | 34 + .../core/service/DictionaryService.java | 92 + .../carbondata/core/service/PathService.java | 38 + .../core/util/CarbonMetadataUtil.java | 114 +- .../carbondata/core/util/CarbonProperties.java | 401 +-- .../org/carbondata/core/util/CarbonUtil.java | 245 +- .../core/util/DataFileFooterConverter.java | 65 +- .../core/writer/CarbonDictionaryWriterImpl.java | 45 +- .../core/writer/CarbonIndexFileWriter.java | 64 + .../core/writer/HierarchyValueWriterForCSV.java | 2 +- .../CarbonDictionarySortIndexWriterImpl.java | 21 +- .../scan/executor/infos/KeyStructureInfo.java | 18 - .../scan/executor/util/QueryUtil.java | 105 +- .../scan/expression/ColumnExpression.java | 3 +- .../carbondata/scan/expression/Expression.java | 4 +- .../scan/expression/ExpressionResult.java | 104 +- .../expression/arithmetic/AddExpression.java | 4 +- .../expression/arithmetic/DivideExpression.java | 4 +- .../arithmetic/MultiplyExpression.java | 4 +- .../arithmetic/SubstractExpression.java | 4 +- .../conditional/EqualToExpression.java | 5 +- .../GreaterThanEqualToExpression.java | 4 +- .../conditional/GreaterThanExpression.java | 5 +- .../expression/conditional/InExpression.java | 6 +- .../conditional/LessThanEqualToExpression.java | 5 +- .../conditional/LessThanExpression.java | 4 +- .../expression/conditional/ListExpression.java | 7 +- .../conditional/NotEqualsExpression.java | 4 +- .../expression/conditional/NotInExpression.java | 6 +- .../exception/FilterIllegalMemberException.java | 98 + .../exception/FilterUnsupportedException.java | 1 - .../scan/expression/logical/AndExpression.java | 6 +- .../scan/expression/logical/NotExpression.java | 4 +- .../scan/expression/logical/OrExpression.java | 4 +- .../scan/filter/FilterExpressionProcessor.java | 36 +- .../org/carbondata/scan/filter/FilterUtil.java | 163 +- .../executer/ColGroupFilterExecuterImpl.java | 190 -- .../ExcludeColGroupFilterExecuterImpl.java | 137 + .../executer/ExcludeFilterExecuterImpl.java | 49 +- .../IncludeColGroupFilterExecuterImpl.java | 209 ++ .../executer/IncludeFilterExecuterImpl.java | 8 +- .../executer/RowLevelFilterExecuterImpl.java | 116 +- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 148 +- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 139 +- ...velRangeLessThanEqualFilterExecuterImpl.java | 187 +- .../RowLevelRangeLessThanFiterExecuterImpl.java | 190 +- .../RowLevelRangeTypeExecuterFacory.java | 13 +- .../filter/resolver/AndFilterResolverImpl.java | 5 +- .../resolver/ConditionalFilterResolverImpl.java | 19 +- .../filter/resolver/FilterResolverIntf.java | 5 +- .../resolver/LogicalFilterResolverImpl.java | 3 +- .../resolver/RestructureFilterResolverImpl.java | 5 +- .../RowLevelRangeFilterResolverImpl.java | 77 +- .../visitor/CustomTypeDictionaryVisitor.java | 16 +- .../visitor/DictionaryColumnVisitor.java | 11 +- .../visitor/NoDictionaryTypeVisitor.java | 12 +- .../scan/result/iterator/RawResultIterator.java | 2 + .../org/carbondata/scan/util/DataTypeUtil.java | 2 +- .../dictionary/AbstractDictionaryCacheTest.java | 7 +- .../dictionary/ForwardDictionaryCacheTest.java | 4 +- .../dictionary/ReverseDictionaryCacheTest.java | 4 +- .../datastore/SegmentTaskIndexStoreTest.java | 143 - ...CarbonDictionarySortIndexReaderImplTest.java | 6 +- .../writer/CarbonDictionaryWriterImplTest.java | 9 +- .../core/writer/CarbonFooterWriterTest.java | 58 +- ...CarbonDictionarySortIndexWriterImplTest.java | 13 +- dev/molap.properties.template | 94 - dev/molap.properties_spark | 90 - .../examples/GenerateDictionaryExample.scala | 4 +- format/src/main/thrift/carbondataindex.thrift | 45 + format/src/main/thrift/schema.thrift | 13 +- .../org/carbondata/hadoop/CarbonPathFilter.java | 4 +- .../AbstractDictionaryDecodedReadSupport.java | 4 +- .../hadoop/test/util/StoreCreator.java | 11 +- .../FACT_UNITED_DATA_INFO_sample_cube.csv | 20 - .../FACT_UNITED_DATA_INFO_sample_table.csv | 20 + .../src/test/resources/restructure_cube.csv | 101 - .../src/test/resources/restructure_table.csv | 101 + .../allqueries/AllDataTypesTestCase1.scala | 82 +- .../allqueries/AllDataTypesTestCase2.scala | 116 +- .../allqueries/AllDataTypesTestCase3.scala | 211 +- .../allqueries/AllDataTypesTestCase4.scala | 2576 +++++------------- .../allqueries/AllDataTypesTestCase5.scala | 165 +- .../allqueries/AllDataTypesTestCase6.scala | 2377 ++++------------ .../spark/merger/RowResultMerger.java | 19 +- .../carbondata/spark/load/CarbonLoadModel.java | 2 + .../carbondata/spark/load/CarbonLoaderUtil.java | 189 +- .../spark/load/DeleteLoadFolders.java | 5 +- .../spark/merger/CarbonDataMergerUtil.java | 401 +-- .../spark/sql/CarbonDatasourceRelation.scala | 22 +- .../spark/sql/CarbonDictionaryDecoder.scala | 7 +- .../org/apache/spark/sql/CarbonSqlParser.scala | 374 +-- .../execution/command/carbonTableSchema.scala | 180 +- .../spark/sql/hive/CarbonMetastoreCatalog.scala | 85 +- .../spark/sql/hive/CarbonStrategies.scala | 4 +- .../spark/CarbonColumnValidator.scala | 36 + .../carbondata/spark/CarbonSparkFactory.scala | 63 + .../spark/DictionaryDetailHelper.scala | 66 + .../spark/rdd/CarbonDataLoadRDD.scala | 4 + .../spark/rdd/CarbonDataRDDFactory.scala | 240 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 7 +- .../carbondata/spark/rdd/CarbonScanRDD.scala | 13 +- .../org/carbondata/spark/rdd/Compactor.scala | 19 +- .../org/carbondata/spark/util/CommonUtil.scala | 79 +- .../spark/util/GlobalDictionaryUtil.scala | 85 +- .../spark/load/CarbonLoaderUtilTest.java | 4 +- .../validation/FileFooterValidator.java | 6 +- integration/spark/src/test/resources/data2.csv | 6 +- .../src/test/resources/data2_DiffTimeFormat.csv | 4 + .../resources/datawithcomplexspecialchar.csv | 151 + .../test/resources/datawithescapecharacter.csv | 22 + .../src/test/resources/datawithnullmeasure.csv | 5 + .../test/resources/datawithspecialcharacter.csv | 38 + .../spark/src/test/resources/filter/emp2.csv | 9 + .../src/test/resources/filter/emp2allnull.csv | 9 + .../src/test/resources/filter/emp2nonull.csv | 8 + .../test/resources/nullandnonparsableValue.csv | 3 + .../apache/spark/sql/TestCarbonSqlParser.scala | 37 +- .../dataload/TestLoadDataWithBlankLine.scala | 18 +- .../TestLoadDataWithEmptyArrayColumns.scala | 12 +- .../TestLoadDataWithMaxMinInteger.scala | 6 +- .../dataload/TestLoadDataWithNullMeasures.scala | 15 +- .../AllDataTypesTestCaseAggregate.scala | 44 +- .../aggquery/IntegerDataTypeTestCase.scala | 7 +- .../AllDataTypesTestCaseAggregate.scala | 8 +- .../testsuite/bigdecimal/TestBigDecimal.scala | 10 + .../bigdecimal/TestNullAndEmptyFields.scala | 110 + .../createtable/TestCreateTableSyntax.scala | 32 +- .../DataCompactionCardinalityBoundryTest.scala | 12 +- .../datacompaction/DataCompactionLockTest.scala | 111 + .../datacompaction/DataCompactionTest.scala | 12 +- .../dataload/TestLoadDataWithHiveSyntax.scala | 108 +- ...ataWithMalformedCarbonCommandException.scala | 4 +- .../dataretention/DataRetentionTestCase.scala | 37 + .../detailquery/AllDataTypesTestCase.scala | 6 +- .../ColumnGroupDataTypesTestCase.scala | 51 +- .../ColumnPropertyValidationTestCase.scala | 28 + .../HighCardinalityDataTypesTestCase.scala | 66 +- .../detailquery/IntegerDataTypeTestCase.scala | 6 +- ...estampDataTypeDirectDictionaryTestCase.scala | 12 +- ...TypeDirectDictionaryWithNoDictTestCase.scala | 4 +- .../filterexpr/AllDataTypesTestCaseFilter.scala | 6 +- .../filterexpr/FilterProcessorTestCase.scala | 131 +- .../GrtLtFilterProcessorTestCase.scala | 176 ++ .../filterexpr/IntegerDataTypeTestCase.scala | 6 +- .../NullMeasureValueTestCaseFilter.scala | 39 + .../HadoopFSRelationTestCase.scala | 16 +- .../joinquery/AllDataTypesTestCaseJoin.scala | 6 +- .../joinquery/IntegerDataTypeTestCase.scala | 6 +- .../NullMeasureValueTestCaseAggregate.scala | 2 +- .../sortexpr/AllDataTypesTestCaseSort.scala | 6 +- .../sortexpr/IntegerDataTypeTestCase.scala | 13 +- .../util/GlobalDictionaryUtilTestCase.scala | 51 +- .../carbonaggregatesurrogategenerator/icon.png | Bin 9058 -> 0 bytes .../plugin.xml | 1 - .../steps/carbonautoagggraphgenerator/icon.png | Bin 9058 -> 0 bytes .../carbonautoagggraphgenerator/plugin.xml | 1 - .../steps/carbonautoaggslicemerger/icon.png | Bin 9058 -> 0 bytes .../steps/carbonautoaggslicemerger/plugin.xml | 1 - .../plugins/steps/carboncsvbasedseqgen/icon.png | Bin 24093 -> 0 bytes .../steps/carboncsvbasedseqgen/plugin.xml | 1 - .../plugins/steps/carboncsvreader/icon.png | Bin 24093 -> 0 bytes .../plugins/steps/carboncsvreader/plugin.xml | 1 - .../plugins/steps/carboncsvreaderstrep/icon.png | Bin 9058 -> 0 bytes .../steps/carboncsvreaderstrep/plugin.xml | 1 - .../steps/carbondatawriter/Binary-icon.png | Bin 14573 -> 0 bytes .../plugins/steps/carbondatawriter/plugin.xml | 1 - .../plugins/steps/carbonfactreader/icon.png | Bin 9058 -> 0 bytes .../plugins/steps/carbonfactreader/plugin.xml | 1 - .../plugins/steps/carbongroupby/icon.png | Bin 9058 -> 0 bytes .../plugins/steps/carbongroupby/plugin.xml | 1 - .../steps/carboninmemoryfactreader/icon.png | Bin 9058 -> 0 bytes .../steps/carboninmemoryfactreader/plugin.xml | 1 - .../.kettle/plugins/steps/carbonseqgen/icon.png | Bin 24093 -> 0 bytes .../plugins/steps/carbonseqgen/plugin.xml | 1 - .../plugins/steps/carbonslicemerger/icon.png | Bin 9058 -> 0 bytes .../plugins/steps/carbonslicemerger/plugin.xml | 1 - .../steps/carbonsortkeyandgroupby/icon.png | Bin 9058 -> 0 bytes .../steps/carbonsortkeyandgroupby/plugin.xml | 1 - .../.kettle/plugins/steps/mdkeygenstep/icon.png | Bin 9058 -> 0 bytes .../plugins/steps/mdkeygenstep/plugin.xml | 1 - .../.kettle/plugins/steps/sortkeystep/icon.png | Bin 9058 -> 0 bytes .../plugins/steps/sortkeystep/plugin.xml | 1 - .../org/carbondata/lcm/locks/LocalFileLock.java | 4 +- .../org/carbondata/lcm/locks/LockUsage.java | 3 +- .../lcm/status/SegmentStatusManager.java | 111 +- .../api/dataloader/DataLoadModel.java | 9 + .../processing/csvload/DataGraphExecuter.java | 8 + .../csvreaderstep/BlockDataHandler.java | 83 +- .../processing/csvreaderstep/CsvInput.java | 4 +- .../processing/csvreaderstep/CsvInputData.java | 2 + .../processing/csvreaderstep/CsvInputMeta.java | 21 +- .../dataprocessor/DataProcessTaskStatus.java | 10 + .../dataprocessor/IDataProcessStatus.java | 2 + .../graphgenerator/GraphGenerator.java | 6 + .../configuration/GraphConfigurationInfo.java | 10 + .../processing/mdkeygen/MDKeyGenStep.java | 40 +- .../merger/step/CarbonSliceMergerStep.java | 3 +- .../processing/schema/metadata/ColumnsInfo.java | 10 + .../store/CarbonFactDataHandlerColumnar.java | 111 +- .../store/CarbonFactDataHandlerModel.java | 63 +- .../store/SingleThreadFinalSortFilesMerger.java | 6 +- .../store/colgroup/ColGroupDataHolder.java | 4 +- .../store/colgroup/ColGroupMinMax.java | 30 +- .../store/writer/AbstractFactDataWriter.java | 147 +- ...actDataWriterImplForIntIndexAndAggBlock.java | 6 +- .../csvbased/CarbonCSVBasedSeqGenMeta.java | 36 + .../csvbased/CarbonCSVBasedSeqGenStep.java | 19 +- .../FileStoreSurrogateKeyGenForCSV.java | 12 +- .../processing/util/CarbonSchemaParser.java | 44 + .../store/colgroup/ColGroupMinMaxTest.java | 385 +-- 250 files changed, 8562 insertions(+), 7434 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java index c177feb,f6f369a..bfa5cdc --- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java +++ b/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java @@@ -32,6 -34,6 +34,7 @@@ import org.carbondata.core.carbon.metad import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; import org.carbondata.core.constants.CarbonCommonConstants; ++import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel; import org.carbondata.core.keygenerator.KeyGenerator; import org.carbondata.core.keygenerator.columnar.ColumnarSplitter; import org.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java index 532c6e9,0000000..26b2519 mode 100644,000000..100644 --- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java +++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java @@@ -1,137 -1,0 +1,116 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.carbondata.core.datastorage.store.columnar; + +public class ColumnGroupModel { + + /** - * cardinality all dimension - */ - private int[] colGroupCardinality; - - /** + * number of columns in columnar block + */ + private int[] columnSplit; + + /** + * total number of columns + */ + private int noOfColumnsStore; + + /** + * whether given index is columnar or not + * true: columnar + * false: row block + */ + private boolean[] columnarStore; + + /** + * column groups + * e.g + * {{0,1,2},3,4,{5,6}} + */ + private int[][] columnGroups; + + /** - * @return cardinality of column groups - */ - public int[] getColumnGroupCardinality() { - return colGroupCardinality; - } - - /** - * set columngroup cardinality - * - * @param columnGroupCardinality - */ - public void setColumnGroupCardinality(int[] columnGroupCardinality) { - this.colGroupCardinality = columnGroupCardinality; - } - - /** + * return columnSplit + * + * @return + */ + public int[] getColumnSplit() { + return columnSplit; + } + + /** + * set columnSplit + * + * @param split + */ + public void setColumnSplit(int[] split) { + this.columnSplit = split; + } + + /** + * @return no of columnar block + */ + public int getNoOfColumnStore() { + return this.noOfColumnsStore; + } + + /** + * set no of columnar block + * + * @param noOfColumnsStore + */ + public void setNoOfColumnStore(int noOfColumnsStore) { + this.noOfColumnsStore = noOfColumnsStore; + } + + /** + * it's an identifier for row block or single column block + * + * @param columnarStore + */ + public void setColumnarStore(boolean[] columnarStore) { + this.columnarStore = columnarStore; + } + + /** + * set column groups + * + * @param columnGroups + */ + public void setColumnGroup(int[][] columnGroups) { + this.columnGroups = columnGroups; + } + + /** + * check if given column group is columnar + * + * @param colGroup + * @return true if given block is columnar + */ + public boolean isColumnar(int colGroup) { + return columnarStore[colGroup]; + } + + /** + * @return columngroups + */ + public int[][] getColumnGroup() { + return this.columnGroups; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java index 8fc5aac,63f8157..e8efa21 --- a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java +++ b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java @@@ -19,35 -19,38 +19,19 @@@ package org.carbondata.core.datastorage.util; - import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.datastorage.store.FileHolder; -import org.carbondata.core.datastorage.store.NodeKeyStore; import org.carbondata.core.datastorage.store.NodeMeasureDataStore; -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStore; -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo; import org.carbondata.core.datastorage.store.compression.ValueCompressionModel; -import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataFileStore; import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore; - -import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataFileStore; import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore; - import org.carbondata.core.util.CarbonProperties; -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarFileKeyStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarFileKeyStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyFileStore; -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyFileStore; -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyInMemoryStore; public final class StoreFactory { /** - * Double array data store. - * key type. -- */ - private static final String COMPRESSED_DOUBLE_ARRAY = "COMPRESSED_DOUBLE_ARRAY"; - private static StoreType keyType; -- /** * value type. */ private static StoreType valueType; static { - String valuetype = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.VALUESTORE_TYPE, - CarbonCommonConstants.VALUESTORE_TYPE_DEFAULT_VAL); - - // set value type - if (COMPRESSED_DOUBLE_ARRAY.equals(valuetype)) { - valueType = StoreType.COMPRESSED_DOUBLE_ARRAY; - } else { - valueType = StoreType.HEAVY_VALUE_COMPRESSION; - } - keyType = StoreType.COMPRESSED_SINGLE_ARRAY; + valueType = StoreType.HEAVY_VALUE_COMPRESSION; } private StoreFactory() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java index 771e5f5,97baf3d..0320621 --- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java +++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java @@@ -39,20 -39,7 +39,14 @@@ import static org.carbondata.core.keyge */ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator { + private TimeStampDirectDictionaryGenerator() { + + } + + public static TimeStampDirectDictionaryGenerator instance = + new TimeStampDirectDictionaryGenerator(); + /** - * Logger instance - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName()); - - /** * The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis. */ public static final long granularityFactor; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/util/CarbonUtil.java index 31d41b3,b19f8d3..2368139 --- a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java @@@ -47,9 -40,14 +40,13 @@@ import org.carbondata.core.carbon.metad import org.carbondata.core.carbon.metadata.datatype.DataType; import org.carbondata.core.carbon.metadata.encoder.Encoding; import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension; + import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; + import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; + import org.carbondata.core.carbon.path.CarbonStorePath; + import org.carbondata.core.carbon.path.CarbonTablePath; import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.datastorage.store.FileHolder; +import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel; import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder; -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo; import org.carbondata.core.datastorage.store.columnar.UnBlockIndexer; import org.carbondata.core.datastorage.store.compression.MeasureMetaDataModel; import org.carbondata.core.datastorage.store.compression.ValueCompressionModel; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java index 6641c18,0000000..52a139c mode 100644,000000..100644 --- a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java +++ b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java @@@ -1,418 -1,0 +1,463 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.carbondata.core.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Iterator; +import java.util.List; + +import org.carbondata.common.logging.LogService; +import org.carbondata.common.logging.LogServiceFactory; ++import org.carbondata.core.carbon.datastore.block.TableBlockInfo; +import org.carbondata.core.carbon.metadata.blocklet.BlockletInfo; +import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter; +import org.carbondata.core.carbon.metadata.blocklet.SegmentInfo; +import org.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta; +import org.carbondata.core.carbon.metadata.blocklet.compressor.CompressionCodec; +import org.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk; +import org.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta; +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletBTreeIndex; +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex; +import org.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex; +import org.carbondata.core.carbon.metadata.blocklet.sort.SortState; +import org.carbondata.core.carbon.metadata.datatype.DataType; +import org.carbondata.core.carbon.metadata.encoder.Encoding; +import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; +import org.carbondata.core.datastorage.store.FileHolder; +import org.carbondata.core.datastorage.store.impl.FileFactory; +import org.carbondata.core.metadata.ValueEncoderMeta; +import org.carbondata.core.reader.CarbonFooterReader; ++import org.carbondata.core.reader.CarbonIndexFileReader; ++import org.carbondata.format.BlockIndex; +import org.carbondata.format.FileFooter; + +/** + * Below class will be used to convert the thrift object of data file + * meta data to wrapper object + */ - class DataFileFooterConverter { ++public class DataFileFooterConverter { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(DataFileFooterConverter.class.getName()); + + /** ++ * Below method will be used to get the index info from index file ++ * ++ * @param filePath file path of the index file ++ * @param tableBlockInfoList table block index ++ * @return list of index info ++ * @throws IOException problem while reading the index file ++ */ ++ public List getIndexInfo(String filePath, List tableBlockInfoList) ++ throws IOException { ++ CarbonIndexFileReader indexReader = new CarbonIndexFileReader(); ++ // open the reader ++ indexReader.openThriftReader(filePath); ++ // get the index header ++ org.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader(); ++ List columnSchemaList = new ArrayList(); ++ List table_columns = readIndexHeader.getTable_columns(); ++ for (int i = 0; i < table_columns.size(); i++) { ++ columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i))); ++ } ++ // get the segment info ++ SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info()); ++ BlockletIndex blockletIndex = null; ++ int counter = 0; ++ DataFileFooter dataFileFooter = null; ++ List dataFileFooters = new ArrayList(); ++ // read the block info from file ++ while (indexReader.hasNext()) { ++ BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo(); ++ blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index()); ++ dataFileFooter = new DataFileFooter(); ++ dataFileFooter.setBlockletIndex(blockletIndex); ++ dataFileFooter.setColumnInTable(columnSchemaList); ++ dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows()); ++ dataFileFooter.setTableBlockInfo(tableBlockInfoList.get(counter++)); ++ dataFileFooter.setSegmentInfo(segmentInfo); ++ dataFileFooters.add(dataFileFooter); ++ } ++ return dataFileFooters; ++ } ++ ++ /** + * Below method will be used to convert thrift file meta to wrapper file meta + */ + public DataFileFooter readDataFileFooter(String filePath, long blockOffset, long blockLength) + throws IOException { + DataFileFooter dataFileFooter = new DataFileFooter(); + FileHolder fileReader = null; + try { + long completeBlockLength = blockOffset + blockLength; + long footerPointer = completeBlockLength - 8; + fileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath)); + long actualFooterOffset = fileReader.readLong(filePath, footerPointer); + CarbonFooterReader reader = new CarbonFooterReader(filePath, actualFooterOffset); + FileFooter footer = reader.readFooter(); + dataFileFooter.setVersionId(footer.getVersion()); + dataFileFooter.setNumberOfRows(footer.getNum_rows()); + dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info())); + List columnSchemaList = new ArrayList(); + List table_columns = footer.getTable_columns(); + for (int i = 0; i < table_columns.size(); i++) { + columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i))); + } + dataFileFooter.setColumnInTable(columnSchemaList); + + List leaf_node_indices_Thrift = + footer.getBlocklet_index_list(); + List blockletIndexList = new ArrayList(); + for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) { + BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i)); + blockletIndexList.add(blockletIndex); + } + + List leaf_node_infos_Thrift = + footer.getBlocklet_info_list(); + List blockletInfoList = new ArrayList(); + for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) { + BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i)); + blockletInfo.setBlockletIndex(blockletIndexList.get(i)); + blockletInfoList.add(blockletInfo); + } + dataFileFooter.setBlockletList(blockletInfoList); + dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList)); + } finally { + if (null != fileReader) { + fileReader.finish(); + } + } + return dataFileFooter; + } + + /** + * Below method will be used to get blocklet index for data file meta + * + * @param blockletIndexList + * @return blocklet index + */ + private BlockletIndex getBlockletIndexForDataFileFooter(List blockletIndexList) { + BlockletIndex blockletIndex = new BlockletIndex(); + BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex(); + blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey()); + blockletBTreeIndex + .setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey()); + blockletIndex.setBtreeIndex(blockletBTreeIndex); + byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone(); + byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone(); + byte[][] minValue = null; + byte[][] maxValue = null; + for (int i = 1; i < blockletIndexList.size(); i++) { + minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues(); + maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues(); + for (int j = 0; j < maxValue.length; j++) { + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) { + currentMinValue[j] = minValue[j].clone(); + } + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) { + currentMaxValue[j] = maxValue[j].clone(); + } + } + } + + BlockletMinMaxIndex minMax = new BlockletMinMaxIndex(); + minMax.setMaxValues(currentMaxValue); + minMax.setMinValues(currentMinValue); + blockletIndex.setMinMaxIndex(minMax); + return blockletIndex; + } + + private ColumnSchema thriftColumnSchmeaToWrapperColumnSchema( + org.carbondata.format.ColumnSchema externalColumnSchema) { + ColumnSchema wrapperColumnSchema = new ColumnSchema(); + wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id()); + wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name()); + wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar()); + wrapperColumnSchema + .setDataType(thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type)); + wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension()); + List encoders = new ArrayList(); + for (org.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) { + encoders.add(fromExternalToWrapperEncoding(encoder)); + } + wrapperColumnSchema.setEncodingList(encoders); + wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child()); + wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision()); + wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); + wrapperColumnSchema.setScale(externalColumnSchema.getScale()); + wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); + wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function()); + return wrapperColumnSchema; + } + + /** + * Below method is to convert the blocklet info of the thrift to wrapper + * blocklet info + * + * @param blockletInfoThrift blocklet info of the thrift + * @return blocklet info wrapper + */ + private BlockletInfo getBlockletInfo(org.carbondata.format.BlockletInfo blockletInfoThrift) { + BlockletInfo blockletInfo = new BlockletInfo(); + List dimensionColumnChunk = new ArrayList(); + List measureChunk = new ArrayList(); + Iterator column_data_chunksIterator = + blockletInfoThrift.getColumn_data_chunksIterator(); - while (column_data_chunksIterator.hasNext()) { - org.carbondata.format.DataChunk next = column_data_chunksIterator.next(); - if (next.isRowMajor()) { - dimensionColumnChunk.add(getDataChunk(next, false)); - } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) { - measureChunk.add(getDataChunk(next, true)); - } else { - - dimensionColumnChunk.add(getDataChunk(next, false)); ++ if (null != column_data_chunksIterator) { ++ while (column_data_chunksIterator.hasNext()) { ++ org.carbondata.format.DataChunk next = column_data_chunksIterator.next(); ++ if (next.isRowMajor()) { ++ dimensionColumnChunk.add(getDataChunk(next, false)); ++ } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) { ++ measureChunk.add(getDataChunk(next, true)); ++ } else { ++ dimensionColumnChunk.add(getDataChunk(next, false)); ++ } + } + } + blockletInfo.setDimensionColumnChunk(dimensionColumnChunk); + blockletInfo.setMeasureColumnChunk(measureChunk); + blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows()); + return blockletInfo; + } + + /** + * Below method is convert the thrift encoding to wrapper encoding + * + * @param encoderThrift thrift encoding + * @return wrapper encoding + */ + private Encoding fromExternalToWrapperEncoding(org.carbondata.format.Encoding encoderThrift) { + switch (encoderThrift) { + case DICTIONARY: + return Encoding.DICTIONARY; + case DELTA: + return Encoding.DELTA; + case RLE: + return Encoding.RLE; + case INVERTED_INDEX: + return Encoding.INVERTED_INDEX; + case BIT_PACKED: + return Encoding.BIT_PACKED; + case DIRECT_DICTIONARY: + return Encoding.DIRECT_DICTIONARY; + default: + return Encoding.DICTIONARY; + } + } + + /** + * Below method will be used to convert the thrift compression to wrapper + * compression codec + * + * @param compressionCodecThrift + * @return wrapper compression codec + */ + private CompressionCodec getCompressionCodec( + org.carbondata.format.CompressionCodec compressionCodecThrift) { + switch (compressionCodecThrift) { + case SNAPPY: + return CompressionCodec.SNAPPY; + default: + return CompressionCodec.SNAPPY; + } + } + + /** + * Below method will be used to convert thrift segment object to wrapper + * segment object + * + * @param segmentInfo thrift segment info object + * @return wrapper segment info object + */ + private SegmentInfo getSegmentInfo(org.carbondata.format.SegmentInfo segmentInfo) { + SegmentInfo info = new SegmentInfo(); + int[] cardinality = new int[segmentInfo.getColumn_cardinalities().size()]; + for (int i = 0; i < cardinality.length; i++) { + cardinality[i] = segmentInfo.getColumn_cardinalities().get(i); + } + info.setColumnCardinality(cardinality); + info.setNumberOfColumns(segmentInfo.getNum_cols()); + return info; + } + + /** + * Below method will be used to convert the blocklet index of thrift to + * wrapper + * + * @param blockletIndexThrift + * @return blocklet index wrapper + */ + private BlockletIndex getBlockletIndex(org.carbondata.format.BlockletIndex blockletIndexThrift) { + org.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index(); + org.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index(); + return new BlockletIndex( + new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()), + new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values())); + } + + /** + * Below method will be used to convert the thrift compression meta to + * wrapper chunk compression meta + * + * @param chunkCompressionMetaThrift + * @return chunkCompressionMetaWrapper + */ + private ChunkCompressorMeta getChunkCompressionMeta( + org.carbondata.format.ChunkCompressionMeta chunkCompressionMetaThrift) { + ChunkCompressorMeta compressorMeta = new ChunkCompressorMeta(); + compressorMeta + .setCompressor(getCompressionCodec(chunkCompressionMetaThrift.getCompression_codec())); + compressorMeta.setCompressedSize(chunkCompressionMetaThrift.getTotal_compressed_size()); + compressorMeta.setUncompressedSize(chunkCompressionMetaThrift.getTotal_uncompressed_size()); + return compressorMeta; + } + + /** + * Below method will be used to convert the thrift data type to wrapper data + * type + * + * @param dataTypeThrift + * @return dataType wrapper + */ + private DataType thriftDataTyopeToWrapperDataType(org.carbondata.format.DataType dataTypeThrift) { + switch (dataTypeThrift) { + case STRING: + return DataType.STRING; + case SHORT: + return DataType.SHORT; + case INT: + return DataType.INT; + case LONG: + return DataType.LONG; + case DOUBLE: + return DataType.DOUBLE; + case DECIMAL: + return DataType.DECIMAL; + case TIMESTAMP: + return DataType.TIMESTAMP; + case ARRAY: + return DataType.ARRAY; + case STRUCT: + return DataType.STRUCT; + default: + return DataType.STRING; + } + } + + /** + * Below method will be used to convert the thrift presence meta to wrapper + * presence meta + * + * @param presentMetadataThrift + * @return wrapper presence meta + */ + private PresenceMeta getPresenceMeta(org.carbondata.format.PresenceMeta presentMetadataThrift) { + PresenceMeta presenceMeta = new PresenceMeta(); + presenceMeta.setRepresentNullValues(presentMetadataThrift.isRepresents_presence()); + presenceMeta.setBitSet(BitSet.valueOf(presentMetadataThrift.getPresent_bit_stream())); + return presenceMeta; + } + + /** + * Below method will be used to convert the thrift object to wrapper object + * + * @param sortStateThrift + * @return wrapper sort state object + */ + private SortState getSortState(org.carbondata.format.SortState sortStateThrift) { + if (sortStateThrift == org.carbondata.format.SortState.SORT_EXPLICIT) { + return SortState.SORT_EXPLICT; + } else if (sortStateThrift == org.carbondata.format.SortState.SORT_NATIVE) { + return SortState.SORT_NATIVE; + } else { + return SortState.SORT_NONE; + } + } + + /** + * Below method will be used to convert the thrift data chunk to wrapper + * data chunk + * + * @param datachunkThrift + * @return wrapper data chunk + */ + private DataChunk getDataChunk(org.carbondata.format.DataChunk datachunkThrift, + boolean isPresenceMetaPresent) { + DataChunk dataChunk = new DataChunk(); + dataChunk.setColumnUniqueIdList(datachunkThrift.getColumn_ids()); + dataChunk.setDataPageLength(datachunkThrift.getData_page_length()); + dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset()); + if (isPresenceMetaPresent) { + dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence())); + } + dataChunk.setRlePageLength(datachunkThrift.getRle_page_length()); + dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset()); + dataChunk.setRowMajor(datachunkThrift.isRowMajor()); + dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length()); + dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset()); + dataChunk.setSortState(getSortState(datachunkThrift.getSort_state())); + dataChunk.setChunkCompressionMeta(getChunkCompressionMeta(datachunkThrift.getChunk_meta())); + List encodingList = new ArrayList(datachunkThrift.getEncoders().size()); + for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) { + encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i))); + } + dataChunk.setEncoderList(encodingList); + if (encodingList.contains(Encoding.DELTA)) { + List thriftEncoderMeta = datachunkThrift.getEncoder_meta(); + List encodeMetaList = + new ArrayList(thriftEncoderMeta.size()); + for (int i = 0; i < thriftEncoderMeta.size(); i++) { + encodeMetaList.add(deserializeEncoderMeta(thriftEncoderMeta.get(i).array())); + } + dataChunk.setValueEncoderMeta(encodeMetaList); + } + return dataChunk; + } + + /** + * Below method will be used to convert the encode metadata to + * ValueEncoderMeta object + * + * @param encoderMeta + * @return ValueEncoderMeta object + */ + private ValueEncoderMeta deserializeEncoderMeta(byte[] encoderMeta) { + // TODO : should remove the unnecessary fields. + ByteArrayInputStream aos = null; + ObjectInputStream objStream = null; + ValueEncoderMeta meta = null; + try { + aos = new ByteArrayInputStream(encoderMeta); + objStream = new ObjectInputStream(aos); + meta = (ValueEncoderMeta) objStream.readObject(); + } catch (ClassNotFoundException e) { + LOGGER.error(e); + } catch (IOException e) { + CarbonUtil.closeStreams(objStream); + } + return meta; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java index 51874b4,0000000..e19a3e9 mode 100644,000000..100644 --- a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java +++ b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java @@@ -1,125 -1,0 +1,107 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.carbondata.scan.executor.infos; + +import org.carbondata.core.keygenerator.KeyGenerator; + +/** + * Below class will store the structure of the key + * used during query execution + */ +public class KeyStructureInfo { + + /** + * it's actually a latest key generator + * last table block as this key generator will be used to + * to update the mdkey of the older slice with the new slice + */ + private KeyGenerator keyGenerator; + + /** + * mask bytes ranges for the query + */ + private int[] maskByteRanges; + + /** + * masked bytes of the query + */ + private int[] maskedBytes; + + /** + * max key for query execution + */ + private byte[] maxKey; + + /** - * mdkey start index of block - */ - private int blockMdKeyStartOffset; - - /** + * @return the keyGenerator + */ + public KeyGenerator getKeyGenerator() { + return keyGenerator; + } + + /** + * @param keyGenerator the keyGenerator to set + */ + public void setKeyGenerator(KeyGenerator keyGenerator) { + this.keyGenerator = keyGenerator; + } + + /** + * @return the maskByteRanges + */ + public int[] getMaskByteRanges() { + return maskByteRanges; + } + + /** + * @param maskByteRanges the maskByteRanges to set + */ + public void setMaskByteRanges(int[] maskByteRanges) { + this.maskByteRanges = maskByteRanges; + } + + /** + * @return the maskedBytes + */ + public int[] getMaskedBytes() { + return maskedBytes; + } + + /** + * @param maskedBytes the maskedBytes to set + */ + public void setMaskedBytes(int[] maskedBytes) { + this.maskedBytes = maskedBytes; + } + + /** + * @return the maxKey + */ + public byte[] getMaxKey() { + return maxKey; + } + + /** + * @param maxKey the maxKey to set + */ + public void setMaxKey(byte[] maxKey) { + this.maxKey = maxKey; + } + - /** - * @param startOffset - */ - public void setBlockMdKeyStartOffset(int startOffset) { - this.blockMdKeyStartOffset = startOffset; - } - - /** - * @return - */ - public int getBlockMdKeyStartOffset() { - return this.blockMdKeyStartOffset; - } +}