carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [22/22] incubator-carbondata git commit: [CACRBONDATA-440] Update-Delete support for Carbon.This closes #492
Date Fri, 06 Jan 2017 13:57:22 GMT
[CACRBONDATA-440] Update-Delete support for Carbon.This closes #492


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b0750c19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b0750c19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b0750c19

Branch: refs/heads/master
Commit: b0750c192be18eb8a784557880f464912a0ad085
Parents: a2a081d 9b8090b
Author: Venkata Ramana G <ramana.gollamudi@huawei.com>
Authored: Fri Jan 6 19:25:24 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Fri Jan 6 19:25:24 2017 +0530

----------------------------------------------------------------------
 .../BlockletLevelDeleteDeltaDataCache.java      |  46 +
 .../iudprocessor/iuddata/BlockMappingVO.java    |  54 +
 .../iuddata/BlockletDeleteDeltaCacheLoader.java |  80 ++
 .../iuddata/DeleteDeltaCacheLoaderIntf.java     |  27 +
 .../iudprocessor/iuddata/RowCountDetailsVO.java |  68 ++
 .../core/carbon/datastore/BlockIndexStore.java  |  49 +-
 .../core/carbon/datastore/DataRefNode.java      |  11 +
 .../carbon/datastore/SegmentTaskIndexStore.java |  89 +-
 .../datastore/TableSegmentUniqueIdentifier.java |  16 +-
 .../block/SegmentTaskIndexWrapper.java          |  33 +-
 .../carbon/datastore/block/TableBlockInfo.java  |  53 +-
 .../impl/btree/AbstractBTreeLeafNode.java       |  22 +
 .../datastore/impl/btree/BTreeNonLeafNode.java  |  22 +
 .../impl/btree/BlockBTreeLeafNode.java          |   2 +-
 .../core/carbon/metadata/encoder/Encoding.java  |   3 +-
 .../metadata/schema/table/CarbonTable.java      |  54 +-
 .../schema/table/column/CarbonColumn.java       |   8 +
 .../table/column/CarbonImplicitDimension.java   | 179 ++++
 .../core/carbon/path/CarbonTablePath.java       | 276 +++++-
 .../core/constants/CarbonCommonConstants.java   | 124 ++-
 .../core/load/LoadMetadataDetails.java          | 158 ++-
 .../core/memory/HeapMemoryAllocator.java        |   2 +-
 .../reader/CarbonDeleteDeltaFileReader.java     |  44 +
 .../reader/CarbonDeleteDeltaFileReaderImpl.java | 147 +++
 .../reader/CarbonDeleteFilesDataReader.java     | 167 ++++
 .../core/update/CarbonUpdateUtil.java           | 797 +++++++++++++++
 .../core/update/DeleteDeltaBlockDetails.java    |  96 ++
 .../core/update/DeleteDeltaBlockletDetails.java |  85 ++
 .../core/update/SegmentUpdateDetails.java       | 172 ++++
 .../carbondata/core/update/TupleIdEnum.java     |  41 +
 .../apache/carbondata/core/update/UpdateVO.java | 108 ++
 .../core/updatestatus/SegmentStatusManager.java | 636 ++++++++++++
 .../SegmentUpdateStatusManager.java             | 984 ++++++++++++++++++
 .../carbondata/core/util/CarbonProperties.java  |  60 ++
 .../apache/carbondata/core/util/CarbonUtil.java |  42 +
 .../core/writer/CarbonDeleteDeltaWriter.java    |  38 +
 .../writer/CarbonDeleteDeltaWriterImpl.java     | 119 +++
 .../fileoperations/AtomicFileOperations.java    |  33 +
 .../AtomicFileOperationsImpl.java               |  87 ++
 .../fileoperations/FileWriteOperation.java      |  25 +
 .../carbondata/locks/AbstractCarbonLock.java    |  77 ++
 .../carbondata/locks/CarbonLockFactory.java     |  94 ++
 .../apache/carbondata/locks/CarbonLockUtil.java |  64 ++
 .../apache/carbondata/locks/HdfsFileLock.java   | 121 +++
 .../apache/carbondata/locks/ICarbonLock.java    |  40 +
 .../apache/carbondata/locks/LocalFileLock.java  | 164 +++
 .../org/apache/carbondata/locks/LockUsage.java  |  36 +
 .../carbondata/locks/ZooKeeperLocking.java      | 195 ++++
 .../apache/carbondata/locks/ZookeeperInit.java  |  82 ++
 .../impl/DictionaryBasedResultCollector.java    |  28 +-
 .../collector/impl/RawBasedResultCollector.java |  12 +
 .../executor/impl/AbstractQueryExecutor.java    |  18 +-
 .../scan/executor/infos/BlockExecutionInfo.java |  24 +
 .../scan/executor/util/QueryUtil.java           |  24 +-
 .../scan/executor/util/RestructureUtil.java     |  13 +-
 .../scan/filter/DimColumnFilterInfo.java        |  19 +
 .../carbondata/scan/model/QueryModel.java       |  14 +
 .../scan/result/AbstractScannedResult.java      |  56 ++
 .../scan/scanner/AbstractBlockletScanner.java   |  13 +
 .../scan/scanner/impl/FilterScanner.java        |  18 +-
 .../scan/wrappers/ByteArrayWrapper.java         |  19 +
 .../core/load/LoadMetadataDetailsUnitTest.java  |  33 +-
 .../CarbonFormatDirectoryStructureTest.java     |   4 +-
 .../DictionaryBasedResultCollectorTest.java     |   4 +
 .../impl/RawBasedResultCollectorTest.java       |   3 +
 .../scan/executor/util/RestructureUtilTest.java |  10 +
 .../scanner/impl/FilterScannerTest.java         |   8 +
 examples/spark/src/main/resources/data.csv      | 992 +------------------
 .../carbondata/hadoop/CarbonInputFormat.java    | 263 +++--
 .../carbondata/hadoop/CarbonInputSplit.java     |  45 +
 .../hadoop/util/BlockLevelTraverser.java        |  86 ++
 .../hadoop/test/util/StoreCreator.java          |  13 +-
 .../carbondata/spark/load/CarbonLoaderUtil.java |  58 +-
 .../spark/load/DeleteLoadFolders.java           |  43 +-
 .../carbondata/spark/load/FailureCauses.java    |  30 +
 .../spark/merger/CarbonDataMergerUtil.java      | 752 +++++++++++++-
 .../merger/CarbonDataMergerUtilResult.java      |  29 +
 .../carbondata/spark/merger/CompactionType.java |   9 +-
 .../spark/merger/RowResultMerger.java           |  60 +-
 .../carbondata/spark/util/LoadMetadataUtil.java |   4 +-
 .../org/apache/carbondata/api/CarbonStore.scala |  12 +-
 .../org/apache/carbondata/spark/KeyVal.scala    |  39 +-
 .../spark/rdd/CarbonDataLoadRDD.scala           | 140 ++-
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |   2 +-
 .../spark/rdd/CarbonIUDMergerRDD.scala          | 132 +++
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  | 138 ++-
 .../apache/carbondata/spark/rdd/Compactor.scala |  51 +-
 .../spark/rdd/DataManagementFunc.scala          | 138 ++-
 .../spark/rdd/UpdateCoalescedRDD.scala          |  87 ++
 .../carbondata/spark/util/CommonUtil.scala      |   2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   4 +-
 .../execution/command/carbonTableSchema.scala   |  21 +-
 .../spark/CarbonDataFrameWriter.scala           |   3 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        | 226 ++++-
 .../carbondata/spark/util/CarbonSparkUtil.scala |  45 +
 .../carbondata/spark/util/QueryPlanUtil.scala   |  56 ++
 .../spark/sql/CarbonCatalystOperators.scala     |  24 +
 .../org/apache/spark/sql/CarbonContext.scala    |   6 +
 .../spark/sql/CarbonDatasourceRelation.scala    |  18 +-
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |   3 +-
 .../scala/org/apache/spark/sql/CarbonScan.scala |   1 +
 .../org/apache/spark/sql/CarbonSqlParser.scala  | 140 ++-
 .../sql/execution/command/IUDCommands.scala     | 851 ++++++++++++++++
 .../execution/command/carbonTableSchema.scala   | 183 +++-
 .../spark/sql/hive/CarbonAnalysisRules.scala    | 173 ++++
 .../apache/spark/sql/hive/CarbonMetastore.scala |   2 +-
 .../sql/hive/CarbonPreInsertionCasts.scala      |  59 --
 .../spark/sql/hive/CarbonStrategies.scala       |  26 +-
 .../spark/sql/optimizer/CarbonOptimizer.scala   |  58 +-
 .../spark/src/test/resources/IUD/T_Hive1.csv    |  10 +
 .../spark/src/test/resources/IUD/comp1.csv      |  11 +
 .../spark/src/test/resources/IUD/comp2.csv      |  11 +
 .../spark/src/test/resources/IUD/comp3.csv      |  11 +
 .../spark/src/test/resources/IUD/comp4.csv      |  11 +
 .../spark/src/test/resources/IUD/dest.csv       |   6 +
 .../spark/src/test/resources/IUD/other.csv      |   3 +
 .../spark/src/test/resources/IUD/sample.csv     |   4 +
 .../src/test/resources/IUD/sample_updated.csv   |   2 +
 .../spark/src/test/resources/IUD/source2.csv    |   3 +
 .../spark/src/test/resources/IUD/source3.csv    |   7 +
 .../spark/src/test/resources/IUD/update01.csv   |   6 +
 .../InsertIntoCarbonTableTestCase.scala         |  20 +-
 .../CompactionSystemLockFeatureTest.scala       |  23 +-
 .../DataCompactionBlockletBoundryTest.scala     |   3 +-
 .../DataCompactionBoundaryConditionsTest.scala  |   3 +-
 .../DataCompactionCardinalityBoundryTest.scala  |  12 +-
 .../datacompaction/DataCompactionLockTest.scala |  24 +-
 .../DataCompactionMinorThresholdTest.scala      |   8 +-
 .../DataCompactionNoDictionaryTest.scala        |   7 +-
 .../datacompaction/DataCompactionTest.scala     |  15 +-
 .../MajorCompactionIgnoreInMinorTest.scala      |  54 +-
 .../MajorCompactionStopsAfterCompaction.scala   |  14 +-
 .../dataretention/DataRetentionTestCase.scala   |   6 +-
 .../ColumnPropertyValidationTestCase.scala      |   7 +-
 .../iud/DeleteCarbonTableTestCase.scala         | 121 +++
 .../testsuite/iud/IUDCompactionTestCases.scala  | 380 +++++++
 .../iud/UpdateCarbonTableTestCase.scala         | 391 ++++++++
 .../spark/rdd/CarbonDataRDDFactory.scala        | 245 ++++-
 .../execution/command/carbonTableSchema.scala   |  57 +-
 .../apache/spark/sql/hive/CarbonMetastore.scala |   4 +-
 .../sql/parser/CarbonSpark2SqlParser.scala      |   3 +-
 .../org/apache/spark/util/Compaction.scala      |   7 +-
 .../fileoperations/AtomicFileOperations.java    |  33 -
 .../AtomicFileOperationsImpl.java               |  87 --
 .../lcm/fileoperations/FileWriteOperation.java  |  25 -
 .../lcm/locks/AbstractCarbonLock.java           |  77 --
 .../carbondata/lcm/locks/CarbonLockFactory.java |  94 --
 .../carbondata/lcm/locks/CarbonLockUtil.java    |  64 --
 .../carbondata/lcm/locks/HdfsFileLock.java      | 120 ---
 .../carbondata/lcm/locks/ICarbonLock.java       |  40 -
 .../carbondata/lcm/locks/LocalFileLock.java     | 164 ---
 .../apache/carbondata/lcm/locks/LockUsage.java  |  35 -
 .../carbondata/lcm/locks/ZooKeeperLocking.java  | 195 ----
 .../carbondata/lcm/locks/ZookeeperInit.java     |  82 --
 .../lcm/status/SegmentStatusManager.java        | 593 -----------
 .../constants/DataProcessorConstants.java       |   5 +
 .../processing/csvreaderstep/CsvInput.java      |  24 +-
 .../csvreaderstep/RddInpututilsForUpdate.java   |  43 +
 .../exception/MultipleMatchingException.java    |  52 +
 .../processing/mdkeygen/MDKeyGenStepMeta.java   |   4 +-
 .../processing/model/CarbonLoadModel.java       |  53 +-
 .../newflow/sort/unsafe/sort/TimSort.java       | 137 ++-
 .../sortdatastep/SortKeyStepMeta.java           |   4 +-
 .../store/CarbonDataFileAttributes.java         |  25 +-
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |   4 +-
 .../FileStoreSurrogateKeyGenForCSV.java         |   6 +-
 .../carbondata/lcm/locks/LocalFileLockTest.java |   2 +
 .../lcm/locks/ZooKeeperLockingTest.java         |  17 +-
 .../carbondata/test/util/StoreCreator.java      |  36 +-
 169 files changed, 11433 insertions(+), 3351 deletions(-)
----------------------------------------------------------------------



Mime
View raw message