carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [6/6] carbondata git commit: [CARBONDATA-1015] Extract interface in data load write step. This closes #971
Date Sat, 10 Jun 2017 03:46:42 GMT
[CARBONDATA-1015] Extract interface in data load write step. This closes #971


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3df5ef81
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3df5ef81
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3df5ef81

Branch: refs/heads/master
Commit: 3df5ef81e4d2253b984847291acc3933306a36b9
Parents: f53ab4b dc83b2a
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Sat Jun 10 09:16:01 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Sat Jun 10 09:16:01 2017 +0530

----------------------------------------------------------------------
 .../core/compression/BigIntCompressor.java      |  34 +-
 .../core/compression/DoubleCompressor.java      |  22 +-
 .../core/compression/ValueCompressor.java       |  75 +---
 .../core/datastore/DimensionType.java           |  35 ++
 .../carbondata/core/datastore/TableSpec.java    | 220 ++++++++++
 .../core/datastore/block/SegmentProperties.java |   6 +-
 .../AbstractMeasureChunkReaderV2V3Format.java   |   2 +
 ...CompressedMeasureChunkFileBasedReaderV2.java |  29 +-
 ...CompressedMeasureChunkFileBasedReaderV3.java |  33 +-
 ...ndexerStorageForNoInvertedIndexForShort.java |   4 +-
 .../compression/MeasureMetaDataModel.java       |   2 +-
 .../compression/WriterCompressModel.java        | 243 -----------
 .../dataholder/CarbonWriteDataHolder.java       | 131 ------
 .../exception/CarbonDataWriterException.java    |  79 ++++
 .../core/datastore/page/ColumnPage.java         | 196 ++++++++-
 .../core/datastore/page/ComplexColumnPage.java  |   8 +-
 .../datastore/page/FixLengthColumnPage.java     | 159 -------
 .../datastore/page/VarLengthColumnPage.java     |  42 --
 .../page/statistics/ColumnPageStatistics.java   | 172 ++++++++
 .../page/statistics/MeasurePageStatsVO.java     | 103 +++++
 .../page/statistics/PageStatistics.java         | 132 ------
 .../core/datastore/row/CarbonRow.java           |  73 ++++
 .../core/datastore/row/WriteStepRowUtil.java    |  86 ++++
 .../core/metadata/BlockletInfoColumnar.java     |  20 +-
 .../core/metadata/CarbonMetadata.java           |   6 +-
 .../core/metadata/datatype/DataType.java        |   5 +-
 .../schema/table/column/CarbonDimension.java    |  23 +-
 .../table/column/CarbonImplicitDimension.java   |   9 +-
 .../schema/table/column/ColumnSchema.java       |   4 +-
 .../executor/impl/AbstractQueryExecutor.java    |   4 +-
 .../core/scan/executor/util/QueryUtil.java      |  22 +-
 .../scan/executor/util/RestructureUtil.java     |   4 +-
 .../util/AbstractDataFileFooterConverter.java   |   2 +-
 .../apache/carbondata/core/util/ByteUtil.java   |  18 +
 .../core/util/CarbonMetadataUtil.java           | 113 ++---
 .../apache/carbondata/core/util/CarbonUtil.java |  43 +-
 .../apache/carbondata/core/util/NodeHolder.java |  61 ++-
 .../core/util/ValueCompressionUtil.java         |  56 +--
 ...ressedDimensionChunkFileBasedReaderTest.java | 132 ------
 ...mpressedMeasureChunkFileBasedReaderTest.java |  90 ----
 .../core/metadata/CarbonMetadataTest.java       |   2 +-
 .../core/util/CarbonMetadataUtilTest.java       |  29 +-
 .../carbondata/core/util/CarbonUtilTest.java    |   5 +-
 .../core/util/ValueCompressionUtilTest.java     | 134 ------
 .../core/writer/CarbonFooterWriterTest.java     |  30 +-
 docs/useful-tips-on-carbondata.md               |   3 +-
 .../util/ExternalColumnDictionaryTestCase.scala |  11 +-
 .../vectorreader/AddColumnTestCases.scala       |   1 +
 .../core/datastore/GenericDataType.java         | 145 +++++++
 .../columnar/ColGroupBlockStorage.java          | 103 +++++
 .../processing/datatypes/ArrayDataType.java     |   1 +
 .../processing/datatypes/GenericDataType.java   | 145 -------
 .../processing/datatypes/PrimitiveDataType.java |   1 +
 .../processing/datatypes/StructDataType.java    |   1 +
 .../merger/CompactionResultSortProcessor.java   |   4 +-
 .../merger/RowResultMergerProcessor.java        |   6 +-
 .../newflow/AbstractDataLoadProcessorStep.java  |   2 +-
 .../newflow/CarbonDataLoadConfiguration.java    |  12 +
 .../newflow/DataLoadProcessBuilder.java         |   3 +
 .../newflow/converter/FieldConverter.java       |   2 +-
 .../newflow/converter/RowConverter.java         |   2 +-
 .../impl/ComplexFieldConverterImpl.java         |   4 +-
 .../impl/DictionaryFieldConverterImpl.java      |   6 +-
 .../DirectDictionaryFieldConverterImpl.java     |  10 +-
 .../converter/impl/FieldEncoderFactory.java     |   2 +-
 .../impl/MeasureFieldConverterImpl.java         |   4 +-
 .../impl/NonDictionaryFieldConverterImpl.java   |   7 +-
 .../converter/impl/RowConverterImpl.java        |   2 +-
 .../processing/newflow/row/CarbonRow.java       |  73 ----
 .../processing/newflow/row/CarbonRowBatch.java  |   1 +
 .../processing/newflow/row/CarbonSortBatch.java |   1 +
 .../newflow/row/WriteStepRowUtil.java           |  86 ----
 .../sort/impl/ParallelReadMergeSorterImpl.java  |   8 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |  10 +-
 .../UnsafeBatchParallelReadMergeSorterImpl.java |   4 +-
 .../impl/UnsafeParallelReadMergeSorterImpl.java |  10 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |  11 +-
 .../newflow/sort/unsafe/IntPointerBuffer.java   |   1 -
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |   2 +-
 .../UnsafeSingleThreadFinalSortFilesMerger.java |   2 +-
 .../CarbonRowDataWriterProcessorStepImpl.java   |   6 +-
 .../steps/DataConverterProcessorStepImpl.java   |   2 +-
 ...ConverterProcessorWithBucketingStepImpl.java |   2 +-
 .../steps/DataWriterBatchProcessorStepImpl.java |   2 +-
 .../steps/DataWriterProcessorStepImpl.java      |   4 +-
 .../newflow/steps/DummyClassForTest.java        |   2 +-
 .../newflow/steps/InputProcessorStepImpl.java   |   2 +-
 .../newflow/steps/SortProcessorStepImpl.java    |   2 +-
 .../sortandgroupby/sortdata/RowComparator.java  |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java    | 429 ++-----------------
 .../store/CarbonFactDataHandlerModel.java       |  32 +-
 .../processing/store/CarbonFactHandler.java     |   4 +-
 .../processing/store/DefaultEncoder.java        | 265 ++++++++++++
 .../store/SingleThreadFinalSortFilesMerger.java |   2 +-
 .../carbondata/processing/store/TablePage.java  |  58 ++-
 .../processing/store/TablePageKey.java          | 139 ++++++
 .../processing/store/TablePageStatistics.java   | 142 ++++++
 .../store/colgroup/ColGroupBlockStorage.java    | 101 -----
 .../store/writer/AbstractFactDataWriter.java    |  57 +--
 .../store/writer/CarbonDataWriterVo.java        |  30 +-
 .../store/writer/CarbonFactDataWriter.java      |  26 +-
 .../processing/store/writer/Encoder.java        |  38 ++
 .../exception/CarbonDataWriterException.java    |  79 ----
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  67 ++-
 .../writer/v2/CarbonFactDataWriterImplV2.java   |  18 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   | 135 +++---
 .../util/CarbonDataProcessorUtil.java           |   2 +-
 .../processing/util/NonDictionaryUtil.java      |   2 +-
 108 files changed, 2385 insertions(+), 2649 deletions(-)
----------------------------------------------------------------------



Mime
View raw message