carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [3/3] incubator-carbondata git commit: [CARBONDATA-742] Added batch sort to improve the loading performance This closes #620
Date Mon, 20 Mar 2017 01:26:24 GMT
[CARBONDATA-742] Added batch sort to improve the loading performance This closes #620


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8d0a672b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8d0a672b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8d0a672b

Branch: refs/heads/master
Commit: 8d0a672b9b2372fd2caffe84c07ddb6ce6b21ece
Parents: 0b44d0e b13ead9
Author: jackylk <jacky.likun@huawei.com>
Authored: Mon Mar 20 09:26:10 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Mar 20 09:26:10 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  19 +
 .../core/datastore/block/TableBlockInfo.java    |   9 +-
 .../carbondata/core/memory/CarbonUnsafe.java    |   4 +
 .../core/mutate/CarbonUpdateUtil.java           |   3 +-
 .../apache/carbondata/core/util/ByteUtil.java   | 136 +++---
 .../core/util/path/CarbonTablePath.java         |  17 +-
 .../datastore/block/TableBlockInfoTest.java     |   2 +-
 .../CarbonFormatDirectoryStructureTest.java     |   4 +-
 .../spark/merger/TupleConversionAdapter.java    |   4 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        |   2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        |   2 +-
 .../graphgenerator/GraphGenerator.java          |   8 +-
 .../processing/mdkeygen/MDKeyGenStep.java       |  18 +-
 .../newflow/CarbonDataLoadConfiguration.java    |  11 +
 .../newflow/DataLoadProcessBuilder.java         |  24 +
 .../constants/DataLoadProcessorConstants.java   |   8 -
 .../converter/DictionaryCardinalityFinder.java  |  26 +
 .../newflow/converter/RowConverter.java         |   2 +-
 .../converter/impl/RowConverterImpl.java        |  32 +-
 .../processing/newflow/row/CarbonRowBatch.java  |   4 +-
 .../processing/newflow/row/CarbonSortBatch.java |  44 ++
 .../UnsafeBatchParallelReadMergeSorterImpl.java | 270 +++++++++++
 .../sort/unsafe/UnsafeCarbonRowPage.java        |   3 +-
 .../sort/unsafe/UnsafeMemoryManager.java        |   4 +
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |  30 +-
 .../unsafe/comparator/UnsafeRowComparator.java  |  34 +-
 .../merger/UnsafeIntermediateFileMerger.java    |   3 +-
 .../UnsafeSingleThreadFinalSortFilesMerger.java |  10 +-
 .../steps/DataConverterProcessorStepImpl.java   |   1 +
 ...ConverterProcessorWithBucketingStepImpl.java |   1 +
 .../steps/DataWriterBatchProcessorStepImpl.java | 191 ++++++++
 .../steps/DataWriterProcessorStepImpl.java      |  28 +-
 .../newflow/steps/SortProcessorStepImpl.java    |   9 +-
 .../sortdata/IntermediateFileMerger.java        |  26 +-
 .../sortandgroupby/sortdata/RowComparator.java  |  10 +-
 .../sortdata/RowComparatorForNormalDims.java    |   6 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |  14 +-
 .../sortdata/SortTempFileChunkHolder.java       |  14 +-
 .../UnCompressedTempSortFileWriter.java         |   8 +-
 .../sortdatastep/SortKeyStep.java               |   8 +-
 .../store/CarbonFactDataHandlerColumnar.java    |  13 +-
 .../store/CarbonFactDataHandlerModel.java       |  13 +-
 .../store/writer/AbstractFactDataWriter.java    |   5 +-
 .../store/writer/CarbonDataWriterVo.java        |  10 +
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |   6 +-
 .../csvbased/CarbonCSVBasedSeqGenStep.java      |   8 +-
 .../processing/util/NonDictionaryUtil.java      | 479 +++++++++++++++++++
 .../processing/util/RemoveDictionaryUtil.java   | 479 -------------------
 48 files changed, 1359 insertions(+), 703 deletions(-)
----------------------------------------------------------------------



Mime
View raw message