carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [09/56] [abbrv] carbondata git commit: [CARBONDATA-1017] Add EncodingStrategy and use ColumnPage for measure encoding and compression. This closes #987
Date Tue, 20 Jun 2017 07:29:15 GMT
[CARBONDATA-1017] Add EncodingStrategy and use ColumnPage for measure encoding and compression.
This closes #987


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/eb9dbfa6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/eb9dbfa6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/eb9dbfa6

Branch: refs/heads/streaming_ingest
Commit: eb9dbfa67f629e4bf26e6c218f8c1b7484551d66
Parents: 3a63bd2 edda248
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Tue Jun 13 22:34:34 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Tue Jun 13 22:34:34 2017 +0530

----------------------------------------------------------------------
 .../dictionary/DoubleArrayTrieDictionary.java   |   4 +-
 .../core/compression/BigIntCompressor.java      | 123 ---
 .../core/compression/DoubleCompressor.java      | 240 ------
 .../core/compression/ValueCompressor.java       |  55 --
 .../core/datastore/MeasureDataWrapper.java      |  28 -
 .../datastore/chunk/MeasureColumnDataChunk.java |  20 +-
 .../measure/AbstractMeasureChunkReader.java     |   4 +
 .../AbstractMeasureChunkReaderV2V3Format.java   |  15 +
 ...CompressedMeasureChunkFileBasedReaderV1.java |  27 +-
 ...CompressedMeasureChunkFileBasedReaderV2.java |  54 +-
 ...CompressedMeasureChunkFileBasedReaderV3.java |  38 +-
 .../compression/MeasureMetaDataModel.java       | 128 ----
 .../compression/ReaderCompressModel.java        |  59 --
 .../compression/ValueCompressionHolder.java     | 118 ---
 .../compression/decimal/CompressByteArray.java  | 107 ---
 .../decimal/CompressionMaxMinByte.java          | 115 ---
 .../decimal/CompressionMaxMinDefault.java       | 118 ---
 .../decimal/CompressionMaxMinInt.java           | 113 ---
 .../decimal/CompressionMaxMinLong.java          | 113 ---
 .../decimal/CompressionMaxMinShort.java         | 116 ---
 .../nondecimal/CompressionNonDecimalByte.java   |  99 ---
 .../CompressionNonDecimalDefault.java           | 102 ---
 .../nondecimal/CompressionNonDecimalInt.java    | 100 ---
 .../nondecimal/CompressionNonDecimalLong.java   | 102 ---
 .../CompressionNonDecimalMaxMinByte.java        | 105 ---
 .../CompressionNonDecimalMaxMinDefault.java     | 107 ---
 .../CompressionNonDecimalMaxMinInt.java         | 106 ---
 .../CompressionNonDecimalMaxMinLong.java        | 106 ---
 .../CompressionNonDecimalMaxMinShort.java       | 105 ---
 .../nondecimal/CompressionNonDecimalShort.java  | 100 ---
 .../compression/none/CompressionNoneByte.java   | 104 ---
 .../none/CompressionNoneDefault.java            | 103 ---
 .../compression/none/CompressionNoneInt.java    | 102 ---
 .../compression/none/CompressionNoneLong.java   | 103 ---
 .../compression/none/CompressionNoneShort.java  | 104 ---
 .../dataholder/CarbonReadDataHolder.java        |  48 --
 .../impl/CompressedDataMeasureDataWrapper.java  |  35 -
 .../core/datastore/page/ColumnPage.java         | 368 ++++++++-
 .../core/datastore/page/ComplexColumnPage.java  |   1 +
 .../core/datastore/page/KeyColumnPage.java      |  42 --
 .../core/datastore/page/LazyColumnPage.java     |  82 ++
 .../core/datastore/page/PrimitiveCodec.java     |  38 +
 .../page/encoding/AdaptiveCompressionCodec.java |  68 ++
 .../page/encoding/AdaptiveIntegerCodec.java     | 206 ++++++
 .../datastore/page/encoding/ColumnCodec.java    |  35 -
 .../page/encoding/ColumnPageCodec.java          |  48 ++
 .../page/encoding/CompressionCodec.java         |  56 ++
 .../page/encoding/DefaultEncodingStrategy.java  | 146 ++++
 .../page/encoding/DeltaIntegerCodec.java        | 229 ++++++
 .../datastore/page/encoding/DummyCodec.java     |  37 -
 .../datastore/page/encoding/EncodedData.java    |  32 +
 .../page/encoding/EncodingStrategy.java         |  71 ++
 .../encoding/UpscaleDeltaFloatingCodec.java     | 187 +++++
 .../page/encoding/UpscaleFloatingCodec.java     | 180 +++++
 .../page/statistics/ColumnPageStatistics.java   | 172 -----
 .../page/statistics/ColumnPageStatsVO.java      | 221 ++++++
 .../page/statistics/MeasurePageStatsVO.java     |   2 +-
 .../core/datastore/row/WriteStepRowUtil.java    |   2 +-
 .../core/memory/IntPointerBuffer.java           | 112 +++
 .../carbondata/core/memory/MemoryException.java |  29 +
 .../core/memory/UnsafeMemoryManager.java        | 149 ++++
 .../core/metadata/datatype/DataType.java        |  46 +-
 .../table/column/CarbonImplicitDimension.java   |   2 +-
 .../schema/table/column/ColumnSchema.java       |   2 +-
 .../impl/AbstractScannedResultCollector.java    |  13 +-
 .../conditional/NotEqualsExpression.java        |   1 -
 .../executer/RowLevelFilterExecuterImpl.java    |  14 +-
 .../core/scan/result/AbstractScannedResult.java |  12 +-
 .../vector/MeasureDataVectorProcessor.java      |  36 +-
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/CompressionFinder.java | 161 ----
 .../core/util/ValueCompressionUtil.java         | 741 -------------------
 .../impl/CompressedDataMeasureWrapperTest.java  |  45 --
 .../DictionaryBasedResultCollectorTest.java     | 152 ----
 .../impl/RawBasedResultCollectorTest.java       |   2 +-
 .../impl/FilterQueryScannedResultTest.java      | 208 ------
 .../impl/NonFilterQueryScannedResultTest.java   | 207 ------
 .../core/util/ValueCompressionUtilTest.java     | 486 ------------
 .../src/test/resources/datawithmaxbigint.csv    |  12 +
 .../src/test/resources/datawithmaxminbigint.csv |  13 +
 .../src/test/resources/datawithminbigint.csv    |  12 +
 .../dataload/TestLoadDataWithMaxMinBigInt.scala |  95 +++
 .../dataload/TestLoadDataWithHiveSyntax.scala   | 702 ------------------
 ...estLoadDataWithHiveSyntaxDefaultFormat.scala | 702 ++++++++++++++++++
 .../TestLoadDataWithHiveSyntaxV1Format.scala    | 709 ++++++++++++++++++
 .../TestLoadDataWithHiveSyntaxV2Format.scala    | 708 ++++++++++++++++++
 .../sql/optimizer/CarbonLateDecodeRule.scala    |   2 +-
 .../UnsafeBatchParallelReadMergeSorterImpl.java |   3 +-
 .../impl/UnsafeParallelReadMergeSorterImpl.java |   3 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |   4 +-
 .../newflow/sort/unsafe/IntPointerBuffer.java   | 116 ---
 .../sort/unsafe/UnsafeCarbonRowPage.java        |   1 +
 .../sort/unsafe/UnsafeMemoryManager.java        | 126 ----
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |  33 +-
 .../unsafe/sort/UnsafeIntSortDataFormat.java    |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java    |  10 +-
 .../processing/store/DefaultEncoder.java        | 265 -------
 .../carbondata/processing/store/TablePage.java  |  28 +-
 .../processing/store/TablePageEncoder.java      | 201 +++++
 .../processing/store/TablePageStatistics.java   |  10 +-
 .../store/writer/CarbonFactDataWriter.java      |   3 +-
 .../processing/store/writer/Encoder.java        |  38 -
 .../writer/v1/CarbonFactDataWriterImplV1.java   |   4 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |   5 +-
 104 files changed, 4824 insertions(+), 6852 deletions(-)
----------------------------------------------------------------------



Mime
View raw message