carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [5/5] incubator-carbondata git commit: [CARBONDATA-2] Data load integration of all steps for removing kettle This closes #263
Date Thu, 10 Nov 2016 14:49:27 GMT
[CARBONDATA-2] Data load integration of all steps for removing kettle This closes #263


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e3085397
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e3085397
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e3085397

Branch: refs/heads/master
Commit: e30853975a986ff15ec5c0823876a5da2f613eef
Parents: ae9d88b 496cde4
Author: jackylk <jacky.likun@huawei.com>
Authored: Thu Nov 10 22:40:26 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Nov 10 22:40:26 2016 +0800

----------------------------------------------------------------------
 .../core/carbon/CarbonTableIdentifier.java      |   8 +
 .../store/dataholder/CarbonWriteDataHolder.java |  42 ++
 .../TimeStampDirectDictionaryGenerator.java     |   7 +-
 .../carbondata/core/load/BlockDetails.java      |  31 +-
 .../carbondata/hadoop/csv/CSVInputFormat.java   |  54 ++
 .../recorditerator/RecordReaderIterator.java    |  68 ++
 .../hadoop/test/util/StoreCreator.java          |   4 +-
 .../sql/common/util/CarbonHiveContext.scala     |   3 +-
 integration/spark/pom.xml                       |   1 +
 .../spark/merger/RowResultMerger.java           |   2 +-
 .../carbondata/spark/load/CarbonLoadModel.java  | 645 ------------------
 .../carbondata/spark/load/CarbonLoaderUtil.java |   1 +
 .../spark/load/DeleteLoadFolders.java           |   1 +
 .../spark/merger/CarbonDataMergerUtil.java      |   2 +-
 .../carbondata/spark/util/LoadMetadataUtil.java |   2 +-
 .../spark/CarbonDataFrameWriter.scala           |   3 +-
 .../spark/rdd/CarbonDataLoadRDD.scala           |   5 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        |  58 +-
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |   3 +-
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |   3 +-
 .../spark/rdd/NewCarbonDataLoadRDD.scala        | 272 ++++++++
 .../spark/util/GlobalDictionaryUtil.scala       |   2 +-
 .../apache/spark/mapred/SparkMapRedUtil.scala   |  32 +
 .../org/apache/spark/sql/CarbonSqlParser.scala  |   3 +-
 .../execution/command/carbonTableSchema.scala   |  59 +-
 .../spark/sql/hive/CarbonStrategies.scala       |   4 +-
 .../org/apache/spark/util/SplitUtils.scala      |   2 +-
 .../TestLoadDataWithNotProperInputFile.scala    |   2 +-
 .../spark/util/AllDictionaryTestCase.scala      |   4 +-
 .../AutoHighCardinalityIdentifyTestCase.scala   |   3 +-
 .../util/ExternalColumnDictionaryTestCase.scala |   3 +-
 ...GlobalDictionaryUtilConcurrentTestCase.scala |   3 +-
 .../util/GlobalDictionaryUtilTestCase.scala     |   4 +-
 .../sql/common/util/CarbonHiveContext.scala     |   3 +-
 pom.xml                                         |   7 +
 .../processing/csvload/DataGraphExecuter.java   |  45 +-
 .../processing/csvload/GraphExecutionUtil.java  |  61 --
 .../processing/datatypes/ArrayDataType.java     |  27 +-
 .../processing/datatypes/GenericDataType.java   |  18 +-
 .../processing/datatypes/PrimitiveDataType.java |  73 ++-
 .../processing/datatypes/StructDataType.java    |  37 +-
 .../processing/mdkeygen/MDKeyGenStep.java       |   2 +-
 .../processing/model/CarbonLoadModel.java       | 647 +++++++++++++++++++
 .../processing/newflow/DataField.java           |  17 +-
 .../processing/newflow/DataLoadExecutor.java    |  72 +++
 .../newflow/DataLoadProcessBuilder.java         | 167 +++++
 .../constants/DataLoadProcessorConstants.java   |   8 +
 .../newflow/converter/BadRecordLogHolder.java   |  46 ++
 .../newflow/converter/FieldConverter.java       |   4 +-
 .../newflow/converter/RowConverter.java         |   4 +
 .../AbstractDictionaryFieldConverterImpl.java   |   4 +-
 .../impl/ComplexFieldConverterImpl.java         |  36 +-
 .../impl/DictionaryFieldConverterImpl.java      |  31 +-
 .../DirectDictionaryFieldConverterImpl.java     |  51 +-
 .../converter/impl/FieldEncoderFactory.java     |  86 ++-
 .../impl/MeasureFieldConverterImpl.java         |  83 +++
 .../impl/NonDictionaryFieldConverterImpl.java   |  20 +-
 .../converter/impl/RowConverterImpl.java        |  50 +-
 .../newflow/dictionary/DirectDictionary.java    |  59 ++
 .../dictionary/PreCreatedDictionary.java        |   8 +-
 .../exception/BadRecordFoundException.java      |  67 ++
 .../newflow/parser/CarbonParserFactory.java     |  25 +-
 .../newflow/parser/GenericParser.java           |   4 +-
 .../newflow/parser/impl/ArrayParserImpl.java    |  33 +-
 .../parser/impl/PrimitiveParserImpl.java        |   2 +-
 .../newflow/parser/impl/RowParserImpl.java      |  68 +-
 .../newflow/parser/impl/StructParserImpl.java   |  27 +-
 .../processing/newflow/row/CarbonRow.java       |  18 +-
 .../sort/impl/ParallelReadMergeSorterImpl.java  |  28 +-
 .../sort/impl/SortPreparatorIterator.java       | 147 -----
 .../steps/DataConverterProcessorStepImpl.java   | 106 ++-
 .../steps/DataWriterProcessorStepImpl.java      | 216 +++++++
 .../newflow/steps/DummyClassForTest.java        |  84 +++
 .../newflow/steps/InputProcessorStepImpl.java   |  47 +-
 .../newflow/steps/SortProcessorStepImpl.java    |   1 +
 .../writer/DataWriterProcessorStepImpl.java     | 222 -------
 .../sortdata/IntermediateFileMerger.java        |  90 ++-
 .../sortdata/NewRowComparator.java              |  73 +++
 .../sortdata/NewRowComparatorForNormalDims.java |  61 ++
 .../sortandgroupby/sortdata/SortDataRows.java   | 112 +++-
 .../sortandgroupby/sortdata/SortParameters.java |  44 +-
 .../sortdata/SortTempFileChunkHolder.java       | 119 +++-
 .../store/CarbonFactDataHandlerColumnar.java    | 309 ++++++++-
 .../store/CarbonFactDataHandlerModel.java       |  40 +-
 .../store/SingleThreadFinalSortFilesMerger.java |   8 +-
 .../csvbased/BadRecordsLogger.java              | 249 +++++++
 .../csvbased/BadRecordslogger.java              | 235 -------
 .../csvbased/CarbonCSVBasedSeqGenStep.java      |  57 +-
 .../util/CarbonDataProcessorUtil.java           | 226 ++++++-
 .../processing/util/RemoveDictionaryUtil.java   |   9 +
 .../carbondata/test/util/StoreCreator.java      |   3 +-
 91 files changed, 4027 insertions(+), 1705 deletions(-)
----------------------------------------------------------------------



Mime
View raw message