Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BA566200CD1 for ; Tue, 20 Jun 2017 09:29:10 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B9A1E160BFA; Tue, 20 Jun 2017 07:29:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D8D1C160BF7 for ; Tue, 20 Jun 2017 09:29:09 +0200 (CEST) Received: (qmail 35540 invoked by uid 500); 20 Jun 2017 07:29:09 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 35412 invoked by uid 99); 20 Jun 2017 07:29:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 20 Jun 2017 07:29:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A075BE152F; Tue, 20 Jun 2017 07:29:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jackylk@apache.org To: commits@carbondata.apache.org Date: Tue, 20 Jun 2017 07:29:15 -0000 Message-Id: <26181883efb4456c93fce32c720b7620@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [09/56] [abbrv] carbondata git commit: [CARBONDATA-1017] Add EncodingStrategy and use ColumnPage for measure encoding and compression. This closes #987 archived-at: Tue, 20 Jun 2017 07:29:10 -0000 [CARBONDATA-1017] Add EncodingStrategy and use ColumnPage for measure encoding and compression. This closes #987 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/eb9dbfa6 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/eb9dbfa6 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/eb9dbfa6 Branch: refs/heads/streaming_ingest Commit: eb9dbfa67f629e4bf26e6c218f8c1b7484551d66 Parents: 3a63bd2 edda248 Author: ravipesala Authored: Tue Jun 13 22:34:34 2017 +0530 Committer: ravipesala Committed: Tue Jun 13 22:34:34 2017 +0530 ---------------------------------------------------------------------- .../dictionary/DoubleArrayTrieDictionary.java | 4 +- .../core/compression/BigIntCompressor.java | 123 --- .../core/compression/DoubleCompressor.java | 240 ------ .../core/compression/ValueCompressor.java | 55 -- .../core/datastore/MeasureDataWrapper.java | 28 - .../datastore/chunk/MeasureColumnDataChunk.java | 20 +- .../measure/AbstractMeasureChunkReader.java | 4 + .../AbstractMeasureChunkReaderV2V3Format.java | 15 + ...CompressedMeasureChunkFileBasedReaderV1.java | 27 +- ...CompressedMeasureChunkFileBasedReaderV2.java | 54 +- ...CompressedMeasureChunkFileBasedReaderV3.java | 38 +- .../compression/MeasureMetaDataModel.java | 128 ---- .../compression/ReaderCompressModel.java | 59 -- .../compression/ValueCompressionHolder.java | 118 --- .../compression/decimal/CompressByteArray.java | 107 --- .../decimal/CompressionMaxMinByte.java | 115 --- .../decimal/CompressionMaxMinDefault.java | 118 --- .../decimal/CompressionMaxMinInt.java | 113 --- .../decimal/CompressionMaxMinLong.java | 113 --- .../decimal/CompressionMaxMinShort.java | 116 --- .../nondecimal/CompressionNonDecimalByte.java | 99 --- .../CompressionNonDecimalDefault.java | 102 --- .../nondecimal/CompressionNonDecimalInt.java | 100 --- .../nondecimal/CompressionNonDecimalLong.java | 102 --- .../CompressionNonDecimalMaxMinByte.java | 105 --- .../CompressionNonDecimalMaxMinDefault.java | 107 --- .../CompressionNonDecimalMaxMinInt.java | 106 --- .../CompressionNonDecimalMaxMinLong.java | 106 --- .../CompressionNonDecimalMaxMinShort.java | 105 --- .../nondecimal/CompressionNonDecimalShort.java | 100 --- .../compression/none/CompressionNoneByte.java | 104 --- .../none/CompressionNoneDefault.java | 103 --- .../compression/none/CompressionNoneInt.java | 102 --- .../compression/none/CompressionNoneLong.java | 103 --- .../compression/none/CompressionNoneShort.java | 104 --- .../dataholder/CarbonReadDataHolder.java | 48 -- .../impl/CompressedDataMeasureDataWrapper.java | 35 - .../core/datastore/page/ColumnPage.java | 368 ++++++++- .../core/datastore/page/ComplexColumnPage.java | 1 + .../core/datastore/page/KeyColumnPage.java | 42 -- .../core/datastore/page/LazyColumnPage.java | 82 ++ .../core/datastore/page/PrimitiveCodec.java | 38 + .../page/encoding/AdaptiveCompressionCodec.java | 68 ++ .../page/encoding/AdaptiveIntegerCodec.java | 206 ++++++ .../datastore/page/encoding/ColumnCodec.java | 35 - .../page/encoding/ColumnPageCodec.java | 48 ++ .../page/encoding/CompressionCodec.java | 56 ++ .../page/encoding/DefaultEncodingStrategy.java | 146 ++++ .../page/encoding/DeltaIntegerCodec.java | 229 ++++++ .../datastore/page/encoding/DummyCodec.java | 37 - .../datastore/page/encoding/EncodedData.java | 32 + .../page/encoding/EncodingStrategy.java | 71 ++ .../encoding/UpscaleDeltaFloatingCodec.java | 187 +++++ .../page/encoding/UpscaleFloatingCodec.java | 180 +++++ .../page/statistics/ColumnPageStatistics.java | 172 ----- .../page/statistics/ColumnPageStatsVO.java | 221 ++++++ .../page/statistics/MeasurePageStatsVO.java | 2 +- .../core/datastore/row/WriteStepRowUtil.java | 2 +- .../core/memory/IntPointerBuffer.java | 112 +++ .../carbondata/core/memory/MemoryException.java | 29 + .../core/memory/UnsafeMemoryManager.java | 149 ++++ .../core/metadata/datatype/DataType.java | 46 +- .../table/column/CarbonImplicitDimension.java | 2 +- .../schema/table/column/ColumnSchema.java | 2 +- .../impl/AbstractScannedResultCollector.java | 13 +- .../conditional/NotEqualsExpression.java | 1 - .../executer/RowLevelFilterExecuterImpl.java | 14 +- .../core/scan/result/AbstractScannedResult.java | 12 +- .../vector/MeasureDataVectorProcessor.java | 36 +- .../apache/carbondata/core/util/CarbonUtil.java | 2 +- .../carbondata/core/util/CompressionFinder.java | 161 ---- .../core/util/ValueCompressionUtil.java | 741 ------------------- .../impl/CompressedDataMeasureWrapperTest.java | 45 -- .../DictionaryBasedResultCollectorTest.java | 152 ---- .../impl/RawBasedResultCollectorTest.java | 2 +- .../impl/FilterQueryScannedResultTest.java | 208 ------ .../impl/NonFilterQueryScannedResultTest.java | 207 ------ .../core/util/ValueCompressionUtilTest.java | 486 ------------ .../src/test/resources/datawithmaxbigint.csv | 12 + .../src/test/resources/datawithmaxminbigint.csv | 13 + .../src/test/resources/datawithminbigint.csv | 12 + .../dataload/TestLoadDataWithMaxMinBigInt.scala | 95 +++ .../dataload/TestLoadDataWithHiveSyntax.scala | 702 ------------------ ...estLoadDataWithHiveSyntaxDefaultFormat.scala | 702 ++++++++++++++++++ .../TestLoadDataWithHiveSyntaxV1Format.scala | 709 ++++++++++++++++++ .../TestLoadDataWithHiveSyntaxV2Format.scala | 708 ++++++++++++++++++ .../sql/optimizer/CarbonLateDecodeRule.scala | 2 +- .../UnsafeBatchParallelReadMergeSorterImpl.java | 3 +- .../impl/UnsafeParallelReadMergeSorterImpl.java | 3 +- ...arallelReadMergeSorterWithBucketingImpl.java | 4 +- .../newflow/sort/unsafe/IntPointerBuffer.java | 116 --- .../sort/unsafe/UnsafeCarbonRowPage.java | 1 + .../sort/unsafe/UnsafeMemoryManager.java | 126 ---- .../newflow/sort/unsafe/UnsafeSortDataRows.java | 33 +- .../unsafe/sort/UnsafeIntSortDataFormat.java | 2 +- .../store/CarbonFactDataHandlerColumnar.java | 10 +- .../processing/store/DefaultEncoder.java | 265 ------- .../carbondata/processing/store/TablePage.java | 28 +- .../processing/store/TablePageEncoder.java | 201 +++++ .../processing/store/TablePageStatistics.java | 10 +- .../store/writer/CarbonFactDataWriter.java | 3 +- .../processing/store/writer/Encoder.java | 38 - .../writer/v1/CarbonFactDataWriterImplV1.java | 4 +- .../writer/v3/CarbonFactDataWriterImplV3.java | 5 +- 104 files changed, 4824 insertions(+), 6852 deletions(-) ----------------------------------------------------------------------