Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 93B2C1749C for ; Wed, 22 Jul 2015 19:52:32 +0000 (UTC) Received: (qmail 13426 invoked by uid 500); 22 Jul 2015 19:52:22 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 13311 invoked by uid 500); 22 Jul 2015 19:52:22 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 10726 invoked by uid 99); 22 Jul 2015 19:52:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jul 2015 19:52:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 320EDE01FB; Wed, 22 Jul 2015 19:52:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jmhsieh@apache.org To: commits@hbase.apache.org Date: Wed, 22 Jul 2015 19:52:36 -0000 Message-Id: In-Reply-To: <050e2ac5b877409d8887a2750c63a417@git.apache.org> References: <050e2ac5b877409d8887a2750c63a417@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/50] [abbrv] hbase git commit: Merge remote-tracking branch 'apache/master' (5/19/15) into hbase-11339 Merge remote-tracking branch 'apache/master' (5/19/15) into hbase-11339 Patches that caused deltas: HBASE-10810 - around HColumnDescriptor 'should' vs 'is' api. HBASE-11677 - LOG was made private HBASE-11927 - Checksum constant changed HBASE-10800 - CellComparator instead of KVComparator Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09a00efc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09a00efc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09a00efc Branch: refs/heads/master Commit: 09a00efc0b37d6a43f684ee30f8b266d27c58c1c Parents: 8a09d95 74e5b26 Author: Jonathan M Hsieh Authored: Tue May 19 16:30:21 2015 -0700 Committer: Jonathan M Hsieh Committed: Tue May 19 16:30:21 2015 -0700 ---------------------------------------------------------------------- .gitignore | 1 + bin/draining_servers.rb | 2 +- bin/hbase-daemon.sh | 2 +- bin/hbase.cmd | 12 +- bin/region_mover.rb | 69 +- bin/region_status.rb | 4 +- conf/hbase-env.cmd | 4 + conf/hbase-env.sh | 4 + dev-support/check_compatibility.sh | 2 +- dev-support/test-patch.properties | 4 +- dev-support/test-patch.sh | 2 +- hbase-assembly/pom.xml | 3 - .../src/main/assembly/hadoop-two-compat.xml | 18 +- hbase-assembly/src/main/assembly/src.xml | 26 + .../org/apache/hadoop/hbase/ClusterStatus.java | 20 - .../apache/hadoop/hbase/HColumnDescriptor.java | 312 ++----- .../org/apache/hadoop/hbase/HRegionInfo.java | 59 +- .../apache/hadoop/hbase/HTableDescriptor.java | 6 +- .../org/apache/hadoop/hbase/ServerLoad.java | 4 +- .../org/apache/hadoop/hbase/client/Admin.java | 160 +++- .../hadoop/hbase/client/AsyncProcess.java | 2 +- .../client/ClientAsyncPrefetchScanner.java | 234 ++++++ .../hadoop/hbase/client/ClientIdGenerator.java | 2 +- .../hadoop/hbase/client/ClientScanner.java | 143 +++- .../hbase/client/ClientSimpleScanner.java | 54 ++ .../hadoop/hbase/client/ClientSmallScanner.java | 17 +- .../hbase/client/ConnectionImplementation.java | 4 +- .../apache/hadoop/hbase/client/HBaseAdmin.java | 785 ++++++++++-------- .../org/apache/hadoop/hbase/client/HTable.java | 41 +- .../hadoop/hbase/client/HTableInterface.java | 10 +- .../apache/hadoop/hbase/client/MetaCache.java | 7 +- .../client/PreemptiveFastFailInterceptor.java | 2 +- .../hbase/client/RegionServerCallable.java | 2 +- .../org/apache/hadoop/hbase/client/Result.java | 13 +- .../hadoop/hbase/client/ResultScanner.java | 6 + .../hbase/client/ReversedClientScanner.java | 2 +- .../hbase/client/RpcRetryingCallerImpl.java | 1 + .../RpcRetryingCallerWithReadReplicas.java | 2 +- .../org/apache/hadoop/hbase/client/Scan.java | 30 +- .../hadoop/hbase/client/ScannerCallable.java | 26 +- .../client/ScannerCallableWithReplicas.java | 12 +- .../hadoop/hbase/client/TableConfiguration.java | 24 +- .../client/UnmodifyableHTableDescriptor.java | 7 +- .../hadoop/hbase/client/ZooKeeperRegistry.java | 2 +- .../apache/hadoop/hbase/filter/FilterList.java | 6 +- .../hadoop/hbase/filter/FuzzyRowFilter.java | 5 +- .../hbase/filter/MultiRowRangeFilter.java | 10 +- .../apache/hadoop/hbase/filter/ParseFilter.java | 6 +- .../hbase/filter/SingleColumnValueFilter.java | 2 +- .../hadoop/hbase/ipc/AbstractRpcClient.java | 3 +- .../org/apache/hadoop/hbase/ipc/AsyncCall.java | 4 +- .../hadoop/hbase/ipc/AsyncRpcChannel.java | 2 +- .../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 7 +- .../hbase/ipc/AsyncServerResponseHandler.java | 4 +- .../hadoop/hbase/ipc/CoprocessorRpcChannel.java | 2 +- .../org/apache/hadoop/hbase/ipc/IPCUtil.java | 1 + .../hbase/ipc/MasterCoprocessorRpcChannel.java | 2 +- .../hbase/ipc/RegionCoprocessorRpcChannel.java | 2 +- .../ipc/RegionServerCoprocessorRpcChannel.java | 2 +- .../apache/hadoop/hbase/ipc/RpcClientImpl.java | 4 + .../hadoop/hbase/protobuf/ProtobufUtil.java | 2 +- .../hadoop/hbase/protobuf/RequestConverter.java | 3 + .../hbase/protobuf/ResponseConverter.java | 22 +- .../hbase/security/HBaseSaslRpcClient.java | 2 +- .../hbase/security/SaslClientHandler.java | 4 +- .../hbase/security/SecureBulkLoadUtil.java | 2 +- .../hbase/security/access/TablePermission.java | 2 +- .../hbase/security/access/UserPermission.java | 2 +- .../token/AuthenticationTokenSelector.java | 2 +- .../hadoop/hbase/security/token/TokenUtil.java | 2 +- .../security/visibility/VisibilityClient.java | 212 +++-- .../hbase/zookeeper/MasterAddressTracker.java | 30 +- .../hbase/zookeeper/RecoverableZooKeeper.java | 6 +- .../hadoop/hbase/zookeeper/ZKLeaderManager.java | 2 +- .../hbase/zookeeper/ZooKeeperNodeTracker.java | 3 +- .../hadoop/hbase/TestHTableDescriptor.java | 2 +- .../hbase/TestInterfaceAudienceAnnotations.java | 27 +- .../hadoop/hbase/client/TestAsyncProcess.java | 3 + .../hbase/client/TestClientNoCluster.java | 4 +- .../hadoop/hbase/client/TestClientScanner.java | 10 + .../client/TestClientSmallReversedScanner.java | 32 +- .../hbase/client/TestClientSmallScanner.java | 34 +- .../hadoop/hbase/client/TestOperation.java | 9 +- .../hbase/client/TestProcedureFuture.java | 17 +- .../apache/hadoop/hbase/ipc/TestIPCUtil.java | 2 +- .../java/org/apache/hadoop/hbase/AuthUtil.java | 2 +- .../org/apache/hadoop/hbase/CellComparator.java | 827 ++++++++++++------- .../java/org/apache/hadoop/hbase/CellUtil.java | 146 +++- .../org/apache/hadoop/hbase/ChoreService.java | 8 +- .../org/apache/hadoop/hbase/HConstants.java | 17 +- .../java/org/apache/hadoop/hbase/KeyValue.java | 50 +- .../apache/hadoop/hbase/KeyValueTestUtil.java | 2 +- .../org/apache/hadoop/hbase/KeyValueUtil.java | 123 ++- .../org/apache/hadoop/hbase/NoTagsKeyValue.java | 37 + .../org/apache/hadoop/hbase/ScheduledChore.java | 8 +- .../java/org/apache/hadoop/hbase/TableName.java | 6 +- .../hadoop/hbase/codec/KeyValueCodec.java | 3 +- .../hbase/codec/KeyValueCodecWithTags.java | 3 +- .../hadoop/hbase/io/BoundedByteBufferPool.java | 4 +- .../hadoop/hbase/io/compress/Compression.java | 2 +- .../io/encoding/BufferedDataBlockEncoder.java | 162 ++-- .../io/encoding/CopyKeyDataBlockEncoder.java | 4 +- .../hbase/io/encoding/DataBlockEncoder.java | 10 +- .../hbase/io/encoding/DiffKeyDeltaEncoder.java | 4 +- .../hbase/io/encoding/FastDiffDeltaEncoder.java | 4 +- .../io/encoding/PrefixKeyDeltaEncoder.java | 4 +- .../hadoop/hbase/io/hfile/HFileContext.java | 3 +- .../hbase/io/hfile/HFileContextBuilder.java | 3 +- .../hadoop/hbase/util/ByteBufferArray.java | 2 +- .../org/apache/hadoop/hbase/util/Bytes.java | 222 ++++- .../apache/hadoop/hbase/util/ChecksumType.java | 95 +-- .../org/apache/hadoop/hbase/util/ClassSize.java | 2 +- .../java/org/apache/hadoop/hbase/util/JVM.java | 9 + .../org/apache/hadoop/hbase/util/Methods.java | 2 +- .../org/apache/hadoop/hbase/util/Sleeper.java | 2 +- .../org/apache/hadoop/hbase/util/Threads.java | 2 +- .../hbase/util/test/RedundantKVGenerator.java | 3 +- .../src/main/resources/hbase-default.xml | 28 +- .../apache/hadoop/hbase/TestCellComparator.java | 100 +-- .../apache/hadoop/hbase/TestChoreService.java | 764 +++++++++-------- .../org/apache/hadoop/hbase/TestKeyValue.java | 147 +--- .../hadoop/hbase/codec/TestCellCodec.java | 7 +- .../hbase/codec/TestCellCodecWithTags.java | 8 +- .../hbase/codec/TestKeyValueCodecWithTags.java | 8 +- .../io/crypto/TestKeyStoreKeyProvider.java | 2 +- .../org/apache/hadoop/hbase/util/TestBytes.java | 41 +- .../apache/hadoop/hbase/types/TestPBCell.java | 3 +- .../hbase/ipc/MetricsHBaseServerSource.java | 22 + .../hbase/master/MetricsMasterSource.java | 4 +- .../MetricsHBaseServerSourceFactoryImpl.java | 6 +- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 71 +- .../hbase/master/MetricsMasterSourceImpl.java | 3 +- .../hadoop/hbase/IntegrationTestIngest.java | 1 + .../IntegrationTestIngestWithEncryption.java | 6 +- ...tegrationTestIngestWithVisibilityLabels.java | 4 +- .../hbase/IntegrationTestMetaReplicas.java | 2 +- .../hadoop/hbase/chaos/actions/Action.java | 2 +- .../actions/RollingBatchRestartRsAction.java | 2 +- .../hadoop/hbase/chaos/policies/Policy.java | 2 +- .../mapreduce/IntegrationTestImportTsv.java | 7 +- .../test/IntegrationTestBigLinkedList.java | 2 +- ...egrationTestBigLinkedListWithVisibility.java | 21 +- ...tionTestWithCellVisibilityLoadAndVerify.java | 16 +- .../hbase/codec/prefixtree/PrefixTreeCodec.java | 18 +- .../codec/prefixtree/PrefixTreeSeeker.java | 12 +- .../decode/PrefixTreeArrayScanner.java | 4 +- .../codec/prefixtree/decode/PrefixTreeCell.java | 25 +- .../codec/prefixtree/row/BaseTestRowData.java | 4 +- .../prefixtree/row/TestPrefixTreeSearcher.java | 6 +- .../row/data/TestRowDataNumberStrings.java | 2 +- .../row/data/TestRowDataSearcherRowMiss.java | 14 +- .../prefixtree/row/data/TestRowDataSimple.java | 14 +- .../hbase/procedure2/OnePhaseProcedure.java | 2 +- .../procedure2/ProcedureAbortedException.java | 2 +- .../hbase/procedure2/ProcedureException.java | 2 +- .../hbase/procedure2/ProcedureExecutor.java | 68 +- .../hbase/procedure2/ProcedureResult.java | 2 +- .../procedure2/ProcedureYieldException.java | 2 +- .../procedure2/RemoteProcedureException.java | 2 +- .../hbase/procedure2/SequentialProcedure.java | 2 +- .../hbase/procedure2/StateMachineProcedure.java | 2 +- .../hbase/procedure2/TwoPhaseProcedure.java | 2 +- .../hbase/procedure2/store/ProcedureStore.java | 2 +- .../procedure2/store/ProcedureStoreTracker.java | 21 +- .../CorruptedWALProcedureStoreException.java | 2 +- .../procedure2/store/wal/WALProcedureStore.java | 134 ++- .../hbase/procedure2/util/StringUtils.java | 4 + .../store/TestProcedureStoreTracker.java | 47 ++ .../hbase/protobuf/generated/ClientProtos.java | 349 ++++++-- .../protobuf/generated/ClusterStatusProtos.java | 100 +-- .../hbase/protobuf/generated/MasterProtos.java | 589 ++++++++----- .../apache/hadoop/hbase/util/ByteStringer.java | 2 +- hbase-protocol/src/main/protobuf/Client.proto | 7 + .../src/main/protobuf/ClusterStatus.proto | 4 +- hbase-protocol/src/main/protobuf/Master.proto | 2 + .../apache/hadoop/hbase/rest/RESTServer.java | 1 + .../apache/hadoop/hbase/rest/RESTServlet.java | 2 +- .../hadoop/hbase/rest/SchemaResource.java | 4 +- .../hadoop/hbase/rest/client/RemoteHTable.java | 5 + .../rest/model/StorageClusterStatusModel.java | 30 +- .../generated/StorageClusterStatusMessage.java | 96 +-- .../protobuf/StorageClusterStatusMessage.proto | 4 +- .../apache/hadoop/hbase/rest/DummyFilter.java | 2 +- .../hbase/rest/HBaseRESTTestingUtility.java | 2 +- .../hbase/rest/TestScannersWithLabels.java | 10 +- .../hadoop/hbase/rest/TestStatusResource.java | 2 +- hbase-server/pom.xml | 1 + .../tmpl/master/BackupMasterStatusTmpl.jamon | 51 +- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 5 +- .../tmpl/regionserver/RegionListTmpl.jamon | 6 +- .../apache/hadoop/hbase/HealthCheckChore.java | 2 +- .../org/apache/hadoop/hbase/HealthChecker.java | 2 +- .../org/apache/hadoop/hbase/JMXListener.java | 18 +- .../apache/hadoop/hbase/LocalHBaseCluster.java | 2 +- .../apache/hadoop/hbase/SplitLogCounters.java | 5 +- .../org/apache/hadoop/hbase/TagRewriteCell.java | 5 + .../apache/hadoop/hbase/ZKNamespaceManager.java | 2 +- .../org/apache/hadoop/hbase/ZNodeClearer.java | 2 +- .../hbase/client/ClientSideRegionScanner.java | 9 +- .../hbase/client/TableSnapshotScanner.java | 5 + .../hadoop/hbase/conf/ConfigurationManager.java | 2 +- .../ForeignExceptionDispatcher.java | 2 +- .../hadoop/hbase/http/HttpRequestLog.java | 2 +- .../apache/hadoop/hbase/http/HttpServer.java | 13 +- .../hadoop/hbase/io/HalfStoreFileReader.java | 26 +- .../hadoop/hbase/io/hfile/ChecksumUtil.java | 110 +-- .../hadoop/hbase/io/hfile/FixedFileTrailer.java | 76 +- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 31 +- .../hadoop/hbase/io/hfile/HFileBlockIndex.java | 50 +- .../hbase/io/hfile/HFilePrettyPrinter.java | 14 +- .../hadoop/hbase/io/hfile/HFileReaderImpl.java | 270 ++++-- .../hbase/io/hfile/HFileWriterFactory.java | 4 +- .../hadoop/hbase/io/hfile/HFileWriterImpl.java | 141 +++- .../hadoop/hbase/io/hfile/LruBlockCache.java | 2 +- .../hbase/io/hfile/bucket/BucketAllocator.java | 2 +- .../hbase/io/hfile/bucket/BucketCache.java | 2 +- .../org/apache/hadoop/hbase/ipc/CallRunner.java | 14 +- .../hadoop/hbase/ipc/MetricsHBaseServer.java | 34 + .../apache/hadoop/hbase/ipc/QosPriority.java | 32 + .../hadoop/hbase/ipc/RWQueueRpcExecutor.java | 3 + .../org/apache/hadoop/hbase/ipc/RpcServer.java | 23 +- .../hadoop/hbase/ipc/SimpleRpcScheduler.java | 2 +- .../hbase/mapred/TableRecordReaderImpl.java | 2 +- .../hbase/mapreduce/HFileOutputFormat.java | 2 +- .../hbase/mapreduce/HFileOutputFormat2.java | 9 +- .../hbase/mapreduce/KeyValueSerialization.java | 5 +- .../hbase/mapreduce/KeyValueSortReducer.java | 3 +- .../hbase/mapreduce/LoadIncrementalHFiles.java | 82 +- .../mapreduce/MultiTableInputFormatBase.java | 2 +- .../hadoop/hbase/mapreduce/PutSortReducer.java | 3 +- .../mapreduce/SimpleTotalOrderPartitioner.java | 2 +- .../hbase/mapreduce/TableInputFormatBase.java | 2 +- .../hbase/mapreduce/TableMapReduceUtil.java | 7 +- .../hbase/mapreduce/TableRecordReaderImpl.java | 2 +- .../hadoop/hbase/mapreduce/TableSplit.java | 24 +- .../hadoop/hbase/mapreduce/TextSortReducer.java | 3 +- .../hadoop/hbase/mapreduce/WALPlayer.java | 2 +- .../hadoop/hbase/master/AssignmentManager.java | 90 +- .../master/AssignmentVerificationReport.java | 2 +- .../hbase/master/GeneralBulkAssigner.java | 29 +- .../org/apache/hadoop/hbase/master/HMaster.java | 26 +- .../hadoop/hbase/master/HMasterCommandLine.java | 27 +- .../hadoop/hbase/master/MasterRpcServices.java | 15 +- .../hadoop/hbase/master/MasterServices.java | 4 +- .../hadoop/hbase/master/MetricsMaster.java | 4 +- .../apache/hadoop/hbase/master/RackManager.java | 2 +- .../hbase/master/balancer/FavoredNodesPlan.java | 2 +- .../master/balancer/RegionLocationFinder.java | 2 +- .../hbase/master/cleaner/HFileLinkCleaner.java | 13 +- .../hadoop/hbase/master/cleaner/LogCleaner.java | 2 +- .../master/cleaner/TimeToLiveHFileCleaner.java | 2 +- .../master/cleaner/TimeToLiveLogCleaner.java | 2 +- .../master/handler/ModifyTableHandler.java | 149 ---- .../master/handler/TableAddFamilyHandler.java | 89 -- .../handler/TableDeleteFamilyHandler.java | 90 -- .../handler/TableModifyFamilyHandler.java | 84 -- .../master/handler/TruncateTableHandler.java | 142 ---- .../master/procedure/CreateTableProcedure.java | 5 + .../master/procedure/DeleteTableProcedure.java | 14 +- .../master/procedure/ProcedureSyncWait.java | 12 + .../hbase/master/snapshot/SnapshotManager.java | 37 +- .../org/apache/hadoop/hbase/mob/MobUtils.java | 23 +- .../PartitionedMobFileCompactor.java | 11 +- .../hbase/namespace/NamespaceAuditor.java | 18 +- .../hbase/namespace/NamespaceStateManager.java | 29 +- .../hadoop/hbase/procedure/ProcedureMember.java | 22 +- .../hadoop/hbase/procedure/Subprocedure.java | 9 +- .../procedure/ZKProcedureCoordinatorRpcs.java | 2 +- .../hbase/procedure/ZKProcedureMemberRpcs.java | 18 +- .../hadoop/hbase/quotas/MasterQuotaManager.java | 6 + .../AnnotationReadingPriorityFunction.java | 52 +- .../hbase/regionserver/CellSkipListSet.java | 4 +- .../hbase/regionserver/CompactSplitThread.java | 5 +- .../hbase/regionserver/DefaultMemStore.java | 7 +- .../hbase/regionserver/DefaultStoreEngine.java | 4 +- .../regionserver/DefaultStoreFileManager.java | 8 +- .../hbase/regionserver/DefaultStoreFlusher.java | 2 +- .../GetClosestRowBeforeTracker.java | 17 +- .../hadoop/hbase/regionserver/HMobStore.java | 20 +- .../hadoop/hbase/regionserver/HRegion.java | 257 ++++-- .../hbase/regionserver/HRegionFileSystem.java | 50 +- .../hbase/regionserver/HRegionServer.java | 32 +- .../hadoop/hbase/regionserver/HStore.java | 27 +- ...IncreasingToUpperBoundRegionSplitPolicy.java | 2 +- .../hadoop/hbase/regionserver/KeyValueHeap.java | 12 +- .../hadoop/hbase/regionserver/LogRoller.java | 2 +- .../hadoop/hbase/regionserver/LruHashMap.java | 2 +- .../hbase/regionserver/MemStoreFlusher.java | 2 +- .../MetricsRegionServerWrapperImpl.java | 2 +- .../regionserver/NoLimitScannerContext.java | 28 +- .../hbase/regionserver/RSDumpServlet.java | 22 +- .../hbase/regionserver/RSRpcServices.java | 189 ++++- .../hadoop/hbase/regionserver/Region.java | 4 + .../regionserver/RegionCoprocessorHost.java | 8 +- .../hbase/regionserver/RegionMergeRequest.java | 2 +- .../hbase/regionserver/RegionSplitPolicy.java | 11 + .../regionserver/ReversedKeyValueHeap.java | 10 +- .../regionserver/ReversedRegionScannerImpl.java | 6 +- .../regionserver/ReversedStoreScanner.java | 9 +- .../hadoop/hbase/regionserver/ScanInfo.java | 10 +- .../hbase/regionserver/ScanQueryMatcher.java | 45 +- .../hbase/regionserver/ScannerContext.java | 128 ++- .../hadoop/hbase/regionserver/SplitRequest.java | 2 +- .../apache/hadoop/hbase/regionserver/Store.java | 4 +- .../hadoop/hbase/regionserver/StoreEngine.java | 8 +- .../hadoop/hbase/regionserver/StoreFile.java | 137 +-- .../hbase/regionserver/StoreFileInfo.java | 7 +- .../hbase/regionserver/StoreFileScanner.java | 22 +- .../hadoop/hbase/regionserver/StoreScanner.java | 41 +- .../regionserver/StripeMultiFileWriter.java | 60 +- .../hbase/regionserver/StripeStoreConfig.java | 2 +- .../hbase/regionserver/StripeStoreEngine.java | 6 +- .../regionserver/StripeStoreFileManager.java | 47 +- .../hbase/regionserver/StripeStoreFlusher.java | 2 +- .../compactions/CompactionConfiguration.java | 2 +- .../compactions/CompactionRequest.java | 2 +- .../regionserver/compactions/Compactor.java | 2 +- .../compactions/DefaultCompactor.java | 11 +- .../compactions/RatioBasedCompactionPolicy.java | 4 +- .../compactions/StripeCompactionPolicy.java | 2 +- .../compactions/StripeCompactor.java | 6 +- .../hadoop/hbase/regionserver/wal/FSHLog.java | 8 +- .../hadoop/hbase/regionserver/wal/HLogKey.java | 2 +- .../hbase/regionserver/wal/MetricsWAL.java | 2 +- .../regionserver/wal/MetricsWALEditsReplay.java | 2 +- .../regionserver/wal/ProtobufLogWriter.java | 2 +- .../hbase/regionserver/wal/WALCellCodec.java | 3 +- .../hadoop/hbase/regionserver/wal/WALEdit.java | 11 +- .../hadoop/hbase/regionserver/wal/WALUtil.java | 2 +- .../HBaseInterClusterReplicationEndpoint.java | 1 + .../replication/regionserver/MetricsSource.java | 2 +- .../regionserver/ReplicationSinkManager.java | 10 + .../regionserver/ReplicationSource.java | 2 +- .../regionserver/ReplicationSyncUp.java | 2 +- .../hbase/security/HBaseSaslRpcServer.java | 2 +- .../security/access/AccessControlLists.java | 2 +- .../hbase/security/access/AccessController.java | 4 +- .../HbaseObjectWritableFor96Migration.java | 2 +- .../security/access/SecureBulkLoadEndpoint.java | 2 +- .../hbase/security/access/TableAuthManager.java | 2 +- .../security/access/ZKPermissionWatcher.java | 2 +- .../token/AuthenticationTokenSecretManager.java | 2 +- .../hbase/security/token/TokenProvider.java | 2 +- .../hbase/security/token/ZKSecretWatcher.java | 2 +- .../visibility/VisibilityController.java | 3 +- .../hbase/snapshot/SnapshotReferenceUtil.java | 2 +- .../apache/hadoop/hbase/util/BloomFilter.java | 61 +- .../hadoop/hbase/util/BloomFilterBase.java | 13 - .../hadoop/hbase/util/BloomFilterChunk.java | 322 ++++++++ .../hadoop/hbase/util/BloomFilterFactory.java | 12 +- .../hadoop/hbase/util/BloomFilterUtil.java | 269 ++++++ .../hadoop/hbase/util/BloomFilterWriter.java | 4 - .../util/BoundedConcurrentLinkedQueue.java | 122 +++ .../hadoop/hbase/util/ByteBloomFilter.java | 662 --------------- .../hbase/util/CollectionBackedScanner.java | 14 +- .../hadoop/hbase/util/CompoundBloomFilter.java | 54 +- .../hbase/util/CompoundBloomFilterBase.java | 33 +- .../hbase/util/CompoundBloomFilterWriter.java | 39 +- .../hadoop/hbase/util/CompressionTest.java | 4 +- .../hadoop/hbase/util/ConnectionCache.java | 3 +- .../hadoop/hbase/util/EncryptionTest.java | 2 +- .../org/apache/hadoop/hbase/util/FSUtils.java | 61 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 85 +- .../hadoop/hbase/util/HBaseFsckRepair.java | 8 +- .../org/apache/hadoop/hbase/util/HMerge.java | 11 +- .../org/apache/hadoop/hbase/util/Merge.java | 2 +- .../hadoop/hbase/util/RegionSizeCalculator.java | 9 +- .../hbase/util/RegionSplitCalculator.java | 3 +- .../hadoop/hbase/util/RegionSplitter.java | 4 +- .../org/apache/hadoop/hbase/wal/WALFactory.java | 11 +- .../org/apache/hadoop/hbase/wal/WALKey.java | 2 +- .../apache/hadoop/hbase/wal/WALSplitter.java | 2 +- .../native/src/mlockall_agent/mlockall_agent.c | 64 +- .../hbase-webapps/master/tablesDetailed.jsp | 13 +- .../org/apache/hadoop/hbase/HBaseCluster.java | 1 + .../hadoop/hbase/HBaseTestingUtility.java | 2 +- .../hbase/HFilePerformanceEvaluation.java | 4 +- .../hbase/MapFilePerformanceEvaluation.java | 2 +- .../apache/hadoop/hbase/MetaMockingUtil.java | 2 +- .../apache/hadoop/hbase/MiniHBaseCluster.java | 2 +- .../hadoop/hbase/MultithreadedTestUtil.java | 2 +- .../hadoop/hbase/PerformanceEvaluation.java | 2 +- .../hbase/PerformanceEvaluationCommons.java | 2 +- .../hadoop/hbase/TestGlobalMemStoreSize.java | 2 +- .../hadoop/hbase/TestHBaseTestingUtility.java | 2 +- .../TestHColumnDescriptorDefaultVersions.java | 8 +- .../org/apache/hadoop/hbase/TestIOFencing.java | 2 +- .../apache/hadoop/hbase/TestInfoServers.java | 2 +- .../hadoop/hbase/TestMovedRegionsCleaner.java | 2 +- .../org/apache/hadoop/hbase/TestNamespace.java | 2 +- .../hbase/TestPartialResultsFromClientSide.java | 3 +- .../hadoop/hbase/TestRegionRebalancing.java | 2 +- .../apache/hadoop/hbase/TestSerialization.java | 8 +- .../hadoop/hbase/TestTableDescriptor.java | 2 +- .../apache/hadoop/hbase/TestTagRewriteCell.java | 48 ++ .../org/apache/hadoop/hbase/TestZooKeeper.java | 2 +- .../hadoop/hbase/backup/TestHFileArchiving.java | 2 +- .../apache/hadoop/hbase/client/TestAdmin1.java | 32 +- .../apache/hadoop/hbase/client/TestAdmin2.java | 3 +- .../client/TestClientScannerRPCTimeout.java | 2 +- .../hadoop/hbase/client/TestClientTimeouts.java | 2 +- .../client/TestCloneSnapshotFromClient.java | 3 +- .../hadoop/hbase/client/TestFastFail.java | 9 +- .../hadoop/hbase/client/TestFromClientSide.java | 39 +- .../hbase/client/TestFromClientSide3.java | 28 +- .../hbase/client/TestHTableMultiplexer.java | 2 +- .../client/TestHTableMultiplexerFlushCache.java | 2 +- .../hbase/client/TestMetaWithReplicas.java | 2 +- .../hbase/client/TestMultipleTimestamps.java | 2 +- .../hbase/client/TestReplicaWithCluster.java | 2 +- .../client/TestRestoreSnapshotFromClient.java | 4 +- .../apache/hadoop/hbase/client/TestResult.java | 17 +- .../hadoop/hbase/client/TestScannerTimeout.java | 2 +- .../client/TestScannersFromClientSide.java | 99 +++ .../hadoop/hbase/client/TestSizeFailures.java | 2 +- .../client/TestSnapshotCloneIndependence.java | 73 +- .../hbase/client/TestSnapshotMetadata.java | 2 +- .../hbase/client/TestTimestampsFilter.java | 2 +- .../replication/TestReplicationAdmin.java | 16 +- .../hadoop/hbase/codec/CodecPerformance.java | 2 + .../hbase/codec/TestCellMessageCodec.java | 9 +- .../hbase/conf/TestConfigurationManager.java | 2 +- .../coprocessor/ColumnAggregationEndpoint.java | 2 +- .../ColumnAggregationEndpointNullResponse.java | 2 +- .../ColumnAggregationEndpointWithErrors.java | 2 +- .../coprocessor/SampleRegionWALObserver.java | 2 +- .../hbase/coprocessor/SimpleRegionObserver.java | 2 +- .../coprocessor/TestCoprocessorEndpoint.java | 1 + .../coprocessor/TestCoprocessorInterface.java | 3 +- .../hbase/coprocessor/TestMasterObserver.java | 10 +- .../TestRegionObserverInterface.java | 2 +- .../TestRegionServerCoprocessorEndpoint.java | 38 +- ...gionServerCoprocessorExceptionWithAbort.java | 5 +- .../coprocessor/TestRowProcessorEndpoint.java | 2 +- .../DummyRegionServerEndpointProtos.java | 81 +- .../hbase/filter/FilterTestingCluster.java | 10 +- .../hbase/filter/TestColumnRangeFilter.java | 2 +- .../hbase/filter/TestDependentColumnFilter.java | 5 +- .../apache/hadoop/hbase/filter/TestFilter.java | 9 +- .../hadoop/hbase/filter/TestFilterList.java | 15 +- .../TestFuzzyRowAndColumnRangeFilter.java | 2 +- .../hadoop/hbase/filter/TestFuzzyRowFilter.java | 6 +- .../hbase/filter/TestMultiRowRangeFilter.java | 4 +- .../TestSingleColumnValueExcludeFilter.java | 5 +- .../hadoop/hbase/http/TestGlobalFilter.java | 2 +- .../hadoop/hbase/http/TestHttpServer.java | 2 +- .../hadoop/hbase/http/TestPathFilter.java | 2 +- .../hadoop/hbase/http/TestServletFilter.java | 2 +- .../hbase/http/resource/JerseyResource.java | 2 +- .../hbase/io/TestHalfStoreFileReader.java | 7 +- .../apache/hadoop/hbase/io/TestHeapSize.java | 2 +- .../encoding/TestBufferedDataBlockEncoder.java | 33 + .../hbase/io/encoding/TestChangingEncoding.java | 2 +- .../io/encoding/TestDataBlockEncoders.java | 5 +- .../encoding/TestLoadAndSwitchEncodeOnDisk.java | 2 +- .../io/encoding/TestPrefixTreeEncoding.java | 15 +- .../encoding/TestSeekToBlockWithEncoders.java | 3 +- .../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 3 +- .../hadoop/hbase/io/hfile/TestChecksum.java | 83 +- .../hbase/io/hfile/TestFixedFileTrailer.java | 25 +- .../apache/hadoop/hbase/io/hfile/TestHFile.java | 158 +++- .../hadoop/hbase/io/hfile/TestHFileBlock.java | 6 +- .../hbase/io/hfile/TestHFileBlockIndex.java | 24 +- .../hadoop/hbase/io/hfile/TestHFileSeek.java | 6 +- .../hbase/io/hfile/TestHFileWriterV2.java | 6 +- .../hbase/io/hfile/TestHFileWriterV3.java | 9 +- .../hadoop/hbase/io/hfile/TestPrefetch.java | 3 +- .../hadoop/hbase/io/hfile/TestReseekTo.java | 3 +- .../hadoop/hbase/io/hfile/TestSeekTo.java | 3 +- .../apache/hadoop/hbase/ipc/TestAsyncIPC.java | 2 +- .../hbase/ipc/TestRpcHandlerException.java | 2 +- .../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 18 + .../hbase/ipc/TestSimpleRpcScheduler.java | 2 +- .../hbase/mapreduce/TestHFileOutputFormat.java | 15 +- .../hbase/mapreduce/TestHFileOutputFormat2.java | 9 +- .../hbase/mapreduce/TestImportExport.java | 9 +- .../TestImportTSVWithOperationAttributes.java | 2 +- .../TestImportTSVWithVisibilityLabels.java | 8 +- .../hadoop/hbase/mapreduce/TestImportTsv.java | 2 +- .../TestLoadIncrementalHFilesSplitRecovery.java | 4 +- .../mapreduce/TestMultiTableInputFormat.java | 2 +- .../hadoop/hbase/mapreduce/TestRowCounter.java | 2 +- .../mapreduce/TestTableInputFormatScanBase.java | 2 +- .../hbase/mapreduce/TestWALRecordReader.java | 2 +- .../hadoop/hbase/master/TestCatalogJanitor.java | 11 +- .../master/TestDistributedLogSplitting.java | 3 +- .../TestMasterOperationsForRegionReplicas.java | 2 +- .../hbase/master/TestMasterPriorityRpc.java | 94 +++ .../hadoop/hbase/master/TestMasterShutdown.java | 4 +- .../hbase/master/TestRegionPlacement.java | 2 +- .../hbase/master/TestRegionPlacement2.java | 2 +- .../hadoop/hbase/master/TestRollingRestart.java | 2 + .../hbase/master/TestSplitLogManager.java | 8 +- .../hbase/master/TestTableLockManager.java | 2 +- .../hadoop/hbase/master/TestWarmupRegion.java | 2 +- .../handler/TestTableDeleteFamilyHandler.java | 6 +- .../TestTableDescriptorModification.java | 16 +- .../TestPartitionedMobFileCompactor.java | 9 +- .../hbase/namespace/TestNamespaceAuditor.java | 193 +++++ .../hbase/procedure/TestProcedureManager.java | 2 +- .../procedure/TestZKProcedureControllers.java | 2 +- .../hadoop/hbase/quotas/TestQuotaAdmin.java | 4 +- .../hadoop/hbase/quotas/TestQuotaTableUtil.java | 4 +- .../hbase/regionserver/KeyValueScanFixture.java | 12 +- .../hbase/regionserver/MockStoreFile.java | 4 +- .../hbase/regionserver/TestAtomicOperation.java | 2 +- .../hbase/regionserver/TestBlocksRead.java | 2 +- .../hbase/regionserver/TestCellSkipListSet.java | 3 +- .../hbase/regionserver/TestColumnSeeking.java | 2 +- .../hbase/regionserver/TestCompaction.java | 2 +- .../hbase/regionserver/TestCompactionState.java | 2 +- .../regionserver/TestCompoundBloomFilter.java | 32 +- .../TestCorruptedRegionStoreFile.java | 250 ++++++ .../hbase/regionserver/TestDefaultMemStore.java | 16 +- .../regionserver/TestDefaultStoreEngine.java | 4 +- .../regionserver/TestEncryptionKeyRotation.java | 2 +- .../regionserver/TestFlushRegionEntry.java | 37 +- .../regionserver/TestGetClosestAtOrBefore.java | 4 +- .../hadoop/hbase/regionserver/TestHRegion.java | 14 +- .../regionserver/TestHRegionReplayEvents.java | 82 +- .../regionserver/TestHRegionServerBulkLoad.java | 2 +- .../hbase/regionserver/TestJoinedScanners.java | 2 +- .../hbase/regionserver/TestKeyValueHeap.java | 9 +- .../regionserver/TestKeyValueScanFixture.java | 3 +- .../hbase/regionserver/TestMajorCompaction.java | 2 +- .../regionserver/TestMasterAddressTracker.java | 107 ++- .../hbase/regionserver/TestMinorCompaction.java | 2 +- .../hbase/regionserver/TestMobCompaction.java | 10 +- .../regionserver/TestMultiColumnScanner.java | 4 +- .../hbase/regionserver/TestParallelPut.java | 2 +- .../hbase/regionserver/TestQosFunction.java | 54 +- .../hbase/regionserver/TestQueryMatcher.java | 6 +- .../hbase/regionserver/TestRecoveredEdits.java | 6 +- .../regionserver/TestRegionFavoredNodes.java | 5 +- .../regionserver/TestRegionReplicaFailover.java | 1 + .../regionserver/TestRegionServerHostname.java | 8 +- .../TestRegionServerReportForDuty.java | 18 +- .../regionserver/TestRegionSplitPolicy.java | 4 +- .../regionserver/TestReversibleScanners.java | 5 +- .../hadoop/hbase/regionserver/TestScanner.java | 2 +- .../TestScannerHeartbeatMessages.java | 539 ++++++++++++ .../TestScannerRetriableFailure.java | 171 ++++ .../regionserver/TestSeekOptimizations.java | 5 +- .../TestSplitTransactionOnCluster.java | 22 +- .../hadoop/hbase/regionserver/TestStore.java | 16 +- .../hbase/regionserver/TestStoreFile.java | 30 +- .../hbase/regionserver/TestStoreScanner.java | 13 +- .../hbase/regionserver/TestStripeCompactor.java | 6 +- .../regionserver/TestStripeStoreEngine.java | 4 +- .../TestStripeStoreFileManager.java | 5 +- .../hadoop/hbase/regionserver/TestTags.java | 2 +- .../hbase/regionserver/TestWideScanner.java | 2 +- .../compactions/TestStripeCompactionPolicy.java | 15 +- .../regionserver/wal/SequenceFileLogWriter.java | 2 +- .../hbase/regionserver/wal/TestFSHLog.java | 2 +- .../hbase/regionserver/wal/TestProtobufLog.java | 2 +- .../wal/TestWALActionsListener.java | 4 +- .../hbase/regionserver/wal/TestWALReplay.java | 2 +- .../replication/TestMasterReplication.java | 13 +- .../replication/TestReplicationEndpoint.java | 2 +- .../TestReplicationWALEntryFilters.java | 3 +- .../TestReplicationSinkManager.java | 30 +- .../TestReplicationSourceManager.java | 2 +- .../regionserver/TestReplicationThrottler.java | 27 +- .../apache/hadoop/hbase/security/TestUser.java | 2 +- .../security/access/TestAccessController.java | 7 +- .../security/token/TestTokenAuthentication.java | 4 +- .../security/token/TestZKSecretWatcher.java | 2 +- .../TestDefaultScanLabelGeneratorStack.java | 6 +- .../TestEnforcingScanLabelGenerator.java | 6 +- .../visibility/TestVisibilityLabels.java | 36 +- ...sibilityLabelsOpWithDifferentUsersNoACL.java | 23 +- .../TestVisibilityLabelsReplication.java | 10 +- .../visibility/TestVisibilityLabelsWithACL.java | 43 +- ...ibilityLabelsWithDefaultVisLabelService.java | 21 +- .../TestVisibilityLabelsWithDeletes.java | 18 +- .../TestVisibilityLabelsWithSLGStack.java | 6 +- .../TestVisibilityLablesWithGroups.java | 19 +- .../TestVisibilityWithCheckAuths.java | 12 +- .../TestWithDisabledAuthorization.java | 27 +- .../hbase/snapshot/TestExportSnapshot.java | 3 +- .../TestRestoreFlushSnapshotFromClient.java | 2 +- .../snapshot/TestRestoreSnapshotHelper.java | 2 +- .../apache/hadoop/hbase/util/LoadTestTool.java | 88 +- .../apache/hadoop/hbase/util/MockServer.java | 2 +- .../hadoop/hbase/util/TestBloomFilterChunk.java | 185 +++++ .../util/TestBoundedConcurrentLinkedQueue.java | 161 ++++ .../hadoop/hbase/util/TestByteBloomFilter.java | 169 ---- .../hadoop/hbase/util/TestCompressionTest.java | 2 +- .../hadoop/hbase/util/TestConnectionCache.java | 54 ++ .../hbase/util/TestCoprocessorScanPolicy.java | 2 +- .../apache/hadoop/hbase/util/TestFSUtils.java | 45 +- .../apache/hadoop/hbase/util/TestFSVisitor.java | 2 +- .../apache/hadoop/hbase/util/TestHBaseFsck.java | 15 +- .../apache/hadoop/hbase/util/TestMergeTool.java | 2 +- .../util/hbck/OfflineMetaRebuildTestCore.java | 2 +- .../util/hbck/TestOfflineMetaRebuildBase.java | 5 +- .../util/hbck/TestOfflineMetaRebuildHole.java | 5 +- .../hbck/TestOfflineMetaRebuildOverlap.java | 5 +- .../hbase/wal/TestDefaultWALProvider.java | 2 +- .../apache/hadoop/hbase/wal/TestSecureWAL.java | 2 +- .../apache/hadoop/hbase/wal/TestWALFactory.java | 8 +- .../hbase/wal/TestWALReaderOnSecureWAL.java | 2 +- .../hbase/wal/WALPerformanceEvaluation.java | 2 +- .../hbase/zookeeper/TestZKLeaderManager.java | 2 +- .../protobuf/DummyRegionServerEndpoint.proto | 1 + hbase-shaded/hbase-shaded-client/pom.xml | 61 ++ hbase-shaded/hbase-shaded-server/pom.xml | 61 ++ hbase-shaded/pom.xml | 121 +++ hbase-shell/src/main/ruby/hbase/admin.rb | 15 +- .../src/main/ruby/hbase/visibility_labels.rb | 12 +- .../src/main/ruby/shell/commands/create.rb | 6 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 16 + .../apache/hadoop/hbase/thrift/CallQueue.java | 2 +- .../hbase/thrift/HbaseHandlerMetricsProxy.java | 2 +- .../hadoop/hbase/thrift/IncrementCoalescer.java | 2 +- .../hadoop/hbase/thrift/ThriftHttpServlet.java | 2 +- .../hadoop/hbase/thrift/ThriftServerRunner.java | 2 +- .../hadoop/hbase/thrift/ThriftUtilities.java | 2 +- .../hadoop/hbase/thrift/TestCallQueue.java | 2 +- .../hbase/thrift/TestThriftHttpServer.java | 2 +- .../hbase/thrift/TestThriftServerCmdLine.java | 2 +- .../thrift2/TestThriftHBaseServiceHandler.java | 2 +- ...TestThriftHBaseServiceHandlerWithLabels.java | 10 +- pom.xml | 27 +- .../asciidoc/_chapters/appendix_acl_matrix.adoc | 94 +-- src/main/asciidoc/_chapters/architecture.adoc | 14 + src/main/asciidoc/_chapters/configuration.adoc | 38 +- src/main/asciidoc/_chapters/datamodel.adoc | 2 +- src/main/asciidoc/_chapters/developer.adoc | 20 +- .../asciidoc/_chapters/getting_started.adoc | 10 +- src/main/asciidoc/_chapters/hbase_apis.adoc | 109 ++- src/main/asciidoc/_chapters/mapreduce.adoc | 27 +- src/main/asciidoc/_chapters/ops_mgt.adoc | 187 ++++- src/main/asciidoc/_chapters/schema_design.adoc | 96 ++- src/main/asciidoc/_chapters/upgrading.adoc | 8 +- 636 files changed, 11702 insertions(+), 6145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-common/src/main/resources/hbase-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 17c1ee3,6790c7e..4b8d6b8 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@@ -41,9 -40,9 +41,10 @@@ import org.apache.hadoop.hbase.classifi import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.exceptions.UnknownProtocolException; + import org.apache.hadoop.hbase.ipc.QosPriority; import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.procedure.MasterProcedureManager; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureResult; http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java ---------------------------------------------------------------------- diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index 2159597,0000000..9f08c6e mode 100644,000000..100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@@ -1,751 -1,0 +1,742 @@@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mob; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; - import org.apache.hadoop.hbase.Cell; - import org.apache.hadoop.hbase.HColumnDescriptor; - import org.apache.hadoop.hbase.HConstants; - import org.apache.hadoop.hbase.HRegionInfo; - import org.apache.hadoop.hbase.KeyValue; - import org.apache.hadoop.hbase.TableName; - import org.apache.hadoop.hbase.Tag; - import org.apache.hadoop.hbase.TagType; ++import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.master.TableLockManager; +import org.apache.hadoop.hbase.master.TableLockManager.TableLock; +import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactor; +import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactor; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.StoreFile; - import org.apache.hadoop.hbase.util.Bytes; - import org.apache.hadoop.hbase.util.FSUtils; - import org.apache.hadoop.hbase.util.ReflectionUtils; - import org.apache.hadoop.hbase.util.Threads; ++import org.apache.hadoop.hbase.util.*; + +/** + * The mob utilities + */ +@InterfaceAudience.Private +public class MobUtils { + + private static final Log LOG = LogFactory.getLog(MobUtils.class); + + private static final ThreadLocal LOCAL_FORMAT = + new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("yyyyMMdd"); + } + }; + + /** + * Formats a date to a string. + * @param date The date. + * @return The string format of the date, it's yyyymmdd. + */ + public static String formatDate(Date date) { + return LOCAL_FORMAT.get().format(date); + } + + /** + * Parses the string to a date. + * @param dateString The string format of a date, it's yyyymmdd. + * @return A date. + * @throws ParseException + */ + public static Date parseDate(String dateString) throws ParseException { + return LOCAL_FORMAT.get().parse(dateString); + } + + /** + * Whether the current cell is a mob reference cell. + * @param cell The current cell. + * @return True if the cell has a mob reference tag, false if it doesn't. + */ + public static boolean isMobReferenceCell(Cell cell) { + if (cell.getTagsLength() > 0) { + Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), + TagType.MOB_REFERENCE_TAG_TYPE); + return tag != null; + } + return false; + } + + /** + * Gets the table name tag. + * @param cell The current cell. + * @return The table name tag. + */ + public static Tag getTableNameTag(Cell cell) { + if (cell.getTagsLength() > 0) { + Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), + TagType.MOB_TABLE_NAME_TAG_TYPE); + return tag; + } + return null; + } + + /** + * Whether the tag list has a mob reference tag. + * @param tags The tag list. + * @return True if the list has a mob reference tag, false if it doesn't. + */ + public static boolean hasMobReferenceTag(List tags) { + if (!tags.isEmpty()) { + for (Tag tag : tags) { + if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) { + return true; + } + } + } + return false; + } + + /** + * Indicates whether it's a raw scan. + * The information is set in the attribute "hbase.mob.scan.raw" of scan. + * For a mob cell, in a normal scan the scanners retrieves the mob cell from the mob file. + * In a raw scan, the scanner directly returns cell in HBase without retrieve the one in + * the mob file. + * @param scan The current scan. + * @return True if it's a raw scan. + */ + public static boolean isRawMobScan(Scan scan) { + byte[] raw = scan.getAttribute(MobConstants.MOB_SCAN_RAW); + try { + return raw != null && Bytes.toBoolean(raw); + } catch (IllegalArgumentException e) { + return false; + } + } + + /** + * Indicates whether it's a reference only scan. + * The information is set in the attribute "hbase.mob.scan.ref.only" of scan. + * If it's a ref only scan, only the cells with ref tag are returned. + * @param scan The current scan. + * @return True if it's a ref only scan. + */ + public static boolean isRefOnlyScan(Scan scan) { + byte[] refOnly = scan.getAttribute(MobConstants.MOB_SCAN_REF_ONLY); + try { + return refOnly != null && Bytes.toBoolean(refOnly); + } catch (IllegalArgumentException e) { + return false; + } + } + + /** + * Indicates whether the scan contains the information of caching blocks. + * The information is set in the attribute "hbase.mob.cache.blocks" of scan. + * @param scan The current scan. + * @return True when the Scan attribute specifies to cache the MOB blocks. + */ + public static boolean isCacheMobBlocks(Scan scan) { + byte[] cache = scan.getAttribute(MobConstants.MOB_CACHE_BLOCKS); + try { + return cache != null && Bytes.toBoolean(cache); + } catch (IllegalArgumentException e) { + return false; + } + } + + /** + * Sets the attribute of caching blocks in the scan. + * + * @param scan + * The current scan. + * @param cacheBlocks + * True, set the attribute of caching blocks into the scan, the scanner with this scan + * caches blocks. + * False, the scanner doesn't cache blocks for this scan. + */ + public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) { + scan.setAttribute(MobConstants.MOB_CACHE_BLOCKS, Bytes.toBytes(cacheBlocks)); + } + + /** + * Cleans the expired mob files. + * Cleans the files whose creation date is older than (current - columnFamily.ttl), and + * the minVersions of that column family is 0. + * @param fs The current file system. + * @param conf The current configuration. + * @param tableName The current table name. + * @param columnDescriptor The descriptor of the current column family. + * @param cacheConfig The cacheConfig that disables the block cache. + * @param current The current time. + * @throws IOException + */ + public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, + HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current) + throws IOException { + long timeToLive = columnDescriptor.getTimeToLive(); + if (Integer.MAX_VALUE == timeToLive) { + // no need to clean, because the TTL is not set. + return; + } + + Date expireDate = new Date(current - timeToLive * 1000); + expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate()); + LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!"); + + FileStatus[] stats = null; + Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName); + Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString()); + try { + stats = fs.listStatus(path); + } catch (FileNotFoundException e) { + LOG.warn("Fail to find the mob file " + path, e); + } + if (null == stats) { + // no file found + return; + } + List filesToClean = new ArrayList(); + int deletedFileCount = 0; + for (FileStatus file : stats) { + String fileName = file.getPath().getName(); + try { + MobFileName mobFileName = null; + if (!HFileLink.isHFileLink(file.getPath())) { + mobFileName = MobFileName.create(fileName); + } else { + HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath()); + mobFileName = MobFileName.create(hfileLink.getOriginPath().getName()); + } + Date fileDate = parseDate(mobFileName.getDate()); + if (LOG.isDebugEnabled()) { + LOG.debug("Checking file " + fileName); + } + if (fileDate.getTime() < expireDate.getTime()) { + if (LOG.isDebugEnabled()) { + LOG.debug(fileName + " is an expired file"); + } + filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE)); + } + } catch (Exception e) { + LOG.error("Cannot parse the fileName " + fileName, e); + } + } + if (!filesToClean.isEmpty()) { + try { + removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(), + filesToClean); + deletedFileCount = filesToClean.size(); + } catch (IOException e) { + LOG.error("Fail to delete the mob files " + filesToClean, e); + } + } + LOG.info(deletedFileCount + " expired mob files are deleted"); + } + + /** + * Gets the root dir of the mob files. + * It's {HBASE_DIR}/mobdir. + * @param conf The current configuration. + * @return the root dir of the mob file. + */ + public static Path getMobHome(Configuration conf) { + Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR)); + return new Path(hbaseDir, MobConstants.MOB_DIR_NAME); + } + + /** + * Gets the qualified root dir of the mob files. + * @param conf The current configuration. + * @return The qualified root dir. + * @throws IOException + */ + public static Path getQualifiedMobRootDir(Configuration conf) throws IOException { + Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR)); + Path mobRootDir = new Path(hbaseDir, MobConstants.MOB_DIR_NAME); + FileSystem fs = mobRootDir.getFileSystem(conf); + return mobRootDir.makeQualified(fs); + } + + /** + * Gets the region dir of the mob files. + * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}. + * @param conf The current configuration. + * @param tableName The current table name. + * @return The region dir of the mob files. + */ + public static Path getMobRegionPath(Configuration conf, TableName tableName) { + Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName); + HRegionInfo regionInfo = getMobRegionInfo(tableName); + return new Path(tablePath, regionInfo.getEncodedName()); + } + + /** + * Gets the family dir of the mob files. + * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}. + * @param conf The current configuration. + * @param tableName The current table name. + * @param familyName The current family name. + * @return The family dir of the mob files. + */ + public static Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) { + return new Path(getMobRegionPath(conf, tableName), familyName); + } + + /** + * Gets the family dir of the mob files. + * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}. + * @param regionPath The path of mob region which is a dummy one. + * @param familyName The current family name. + * @return The family dir of the mob files. + */ + public static Path getMobFamilyPath(Path regionPath, String familyName) { + return new Path(regionPath, familyName); + } + + /** + * Gets the HRegionInfo of the mob files. + * This is a dummy region. The mob files are not saved in a region in HBase. + * This is only used in mob snapshot. It's internally used only. + * @param tableName + * @return A dummy mob region info. + */ + public static HRegionInfo getMobRegionInfo(TableName tableName) { + HRegionInfo info = new HRegionInfo(tableName, MobConstants.MOB_REGION_NAME_BYTES, + HConstants.EMPTY_END_ROW, false, 0); + return info; + } + + /** + * Gets whether the current HRegionInfo is a mob one. + * @param regionInfo The current HRegionInfo. + * @return If true, the current HRegionInfo is a mob one. + */ + public static boolean isMobRegionInfo(HRegionInfo regionInfo) { + return regionInfo == null ? false : getMobRegionInfo(regionInfo.getTable()).getEncodedName() + .equals(regionInfo.getEncodedName()); + } + + /** + * Gets whether the current region name follows the pattern of a mob region name. + * @param tableName The current table name. + * @param regionName The current region name. + * @return True if the current region name follows the pattern of a mob region name. + */ + public static boolean isMobRegionName(TableName tableName, byte[] regionName) { + return Bytes.equals(regionName, getMobRegionInfo(tableName).getRegionName()); + } + + /** + * Gets the working directory of the mob compaction. + * @param root The root directory of the mob compaction. + * @param jobName The current job name. + * @return The directory of the mob compaction for the current job. + */ + public static Path getCompactionWorkingPath(Path root, String jobName) { + return new Path(root, jobName); + } + + /** + * Archives the mob files. + * @param conf The current configuration. + * @param fs The current file system. + * @param tableName The table name. + * @param tableDir The table directory. + * @param family The name of the column family. + * @param storeFiles The files to be deleted. + * @throws IOException + */ + public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName, + Path tableDir, byte[] family, Collection storeFiles) throws IOException { + HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family, + storeFiles); + } + + /** + * Creates a mob reference KeyValue. + * The value of the mob reference KeyValue is mobCellValueSize + mobFileName. + * @param cell The original Cell. + * @param fileName The mob file name where the mob reference KeyValue is written. + * @param tableNameTag The tag of the current table name. It's very important in + * cloning the snapshot. + * @return The mob reference KeyValue. + */ + public static KeyValue createMobRefKeyValue(Cell cell, byte[] fileName, Tag tableNameTag) { + // Append the tags to the KeyValue. + // The key is same, the value is the filename of the mob file + List tags = new ArrayList(); + // Add the ref tag as the 1st one. + tags.add(MobConstants.MOB_REF_TAG); + // Add the tag of the source table name, this table is where this mob file is flushed + // from. + // It's very useful in cloning the snapshot. When reading from the cloning table, we need to + // find the original mob files by this table name. For details please see cloning + // snapshot for mob files. + tags.add(tableNameTag); + // Add the existing tags. + tags.addAll(Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength())); + int valueLength = cell.getValueLength(); + byte[] refValue = Bytes.add(Bytes.toBytes(valueLength), fileName); + KeyValue reference = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), + cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), + cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), + cell.getTimestamp(), KeyValue.Type.Put, refValue, 0, refValue.length, tags); + reference.setSequenceId(cell.getSequenceId()); + return reference; + } + + /** + * Creates a writer for the mob file in temp directory. + * @param conf The current configuration. + * @param fs The current file system. + * @param family The descriptor of the current column family. + * @param date The date string, its format is yyyymmmdd. + * @param basePath The basic path for a temp directory. + * @param maxKeyCount The key count. + * @param compression The compression algorithm. + * @param startKey The hex string of the start key. + * @param cacheConfig The current cache config. + * @return The writer for the mob file. + * @throws IOException + */ + public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs, + HColumnDescriptor family, String date, Path basePath, long maxKeyCount, + Compression.Algorithm compression, String startKey, CacheConfig cacheConfig) + throws IOException { + MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString() + .replaceAll("-", "")); + return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression, + cacheConfig); + } + + /** + * Creates a writer for the ref file in temp directory. + * @param conf The current configuration. + * @param fs The current file system. + * @param family The descriptor of the current column family. + * @param basePath The basic path for a temp directory. + * @param maxKeyCount The key count. + * @param cacheConfig The current cache config. + * @return The writer for the mob file. + * @throws IOException + */ + public static StoreFile.Writer createRefFileWriter(Configuration conf, FileSystem fs, + HColumnDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig) + throws IOException { + HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(true) + .withIncludesTags(true).withCompression(family.getCompactionCompression()) - .withCompressTags(family.shouldCompressTags()).withChecksumType(HStore.getChecksumType(conf)) ++ .withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf)) + .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize()) + .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build(); + Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", "")); + StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath) - .withComparator(KeyValue.COMPARATOR).withBloomType(family.getBloomFilterType()) ++ .withComparator(CellComparator.COMPARATOR).withBloomType(family.getBloomFilterType()) + .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build(); + return w; + } + + /** + * Creates a writer for the mob file in temp directory. + * @param conf The current configuration. + * @param fs The current file system. + * @param family The descriptor of the current column family. + * @param date The date string, its format is yyyymmmdd. + * @param basePath The basic path for a temp directory. + * @param maxKeyCount The key count. + * @param compression The compression algorithm. + * @param startKey The start key. + * @param cacheConfig The current cache config. + * @return The writer for the mob file. + * @throws IOException + */ + public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs, + HColumnDescriptor family, String date, Path basePath, long maxKeyCount, + Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig) + throws IOException { + MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString() + .replaceAll("-", "")); + return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression, + cacheConfig); + } + + /** + * Creates a writer for the del file in temp directory. + * @param conf The current configuration. + * @param fs The current file system. + * @param family The descriptor of the current column family. + * @param date The date string, its format is yyyymmmdd. + * @param basePath The basic path for a temp directory. + * @param maxKeyCount The key count. + * @param compression The compression algorithm. + * @param startKey The start key. + * @param cacheConfig The current cache config. + * @return The writer for the del file. + * @throws IOException + */ + public static StoreFile.Writer createDelFileWriter(Configuration conf, FileSystem fs, + HColumnDescriptor family, String date, Path basePath, long maxKeyCount, + Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig) + throws IOException { + String suffix = UUID + .randomUUID().toString().replaceAll("-", "") + "_del"; + MobFileName mobFileName = MobFileName.create(startKey, date, suffix); + return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression, + cacheConfig); + } + + /** + * Creates a writer for the del file in temp directory. + * @param conf The current configuration. + * @param fs The current file system. + * @param family The descriptor of the current column family. + * @param mobFileName The mob file name. + * @param basePath The basic path for a temp directory. + * @param maxKeyCount The key count. + * @param compression The compression algorithm. + * @param cacheConfig The current cache config. + * @return The writer for the mob file. + * @throws IOException + */ + private static StoreFile.Writer createWriter(Configuration conf, FileSystem fs, + HColumnDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount, + Compression.Algorithm compression, CacheConfig cacheConfig) throws IOException { + HFileContext hFileContext = new HFileContextBuilder().withCompression(compression) - .withIncludesMvcc(true).withIncludesTags(true).withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) ++ .withIncludesMvcc(true).withIncludesTags(true) ++ .withChecksumType(ChecksumType.getDefaultChecksumType()) + .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).withBlockSize(family.getBlocksize()) + .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build(); + + StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs) + .withFilePath(new Path(basePath, mobFileName.getFileName())) - .withComparator(KeyValue.COMPARATOR).withBloomType(BloomType.NONE) ++ .withComparator(CellComparator.COMPARATOR).withBloomType(BloomType.NONE) + .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build(); + return w; + } + + /** + * Commits the mob file. + * @param conf The current configuration. + * @param fs The current file system. + * @param sourceFile The path where the mob file is saved. + * @param targetPath The directory path where the source file is renamed to. + * @param cacheConfig The current cache config. + * @return The target file path the source file is renamed to. + * @throws IOException + */ + public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile, + Path targetPath, CacheConfig cacheConfig) throws IOException { + if (sourceFile == null) { + return null; + } + Path dstPath = new Path(targetPath, sourceFile.getName()); + validateMobFile(conf, fs, sourceFile, cacheConfig); + String msg = "Renaming flushed file from " + sourceFile + " to " + dstPath; + LOG.info(msg); + Path parent = dstPath.getParent(); + if (!fs.exists(parent)) { + fs.mkdirs(parent); + } + if (!fs.rename(sourceFile, dstPath)) { + throw new IOException("Failed rename of " + sourceFile + " to " + dstPath); + } + return dstPath; + } + + /** + * Validates a mob file by opening and closing it. + * @param conf The current configuration. + * @param fs The current file system. + * @param path The path where the mob file is saved. + * @param cacheConfig The current cache config. + */ + private static void validateMobFile(Configuration conf, FileSystem fs, Path path, + CacheConfig cacheConfig) throws IOException { + StoreFile storeFile = null; + try { + storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE); + storeFile.createReader(); + } catch (IOException e) { + LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e); + throw e; + } finally { + if (storeFile != null) { + storeFile.closeReader(false); + } + } + } + + /** + * Indicates whether the current mob ref cell has a valid value. + * A mob ref cell has a mob reference tag. + * The value of a mob ref cell consists of two parts, real mob value length and mob file name. + * The real mob value length takes 4 bytes. + * The remaining part is the mob file name. + * @param cell The mob ref cell. + * @return True if the cell has a valid value. + */ + public static boolean hasValidMobRefCellValue(Cell cell) { + return cell.getValueLength() > Bytes.SIZEOF_INT; + } + + /** + * Gets the mob value length from the mob ref cell. + * A mob ref cell has a mob reference tag. + * The value of a mob ref cell consists of two parts, real mob value length and mob file name. + * The real mob value length takes 4 bytes. + * The remaining part is the mob file name. + * @param cell The mob ref cell. + * @return The real mob value length. + */ + public static int getMobValueLength(Cell cell) { + return Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), Bytes.SIZEOF_INT); + } + + /** + * Gets the mob file name from the mob ref cell. + * A mob ref cell has a mob reference tag. + * The value of a mob ref cell consists of two parts, real mob value length and mob file name. + * The real mob value length takes 4 bytes. + * The remaining part is the mob file name. + * @param cell The mob ref cell. + * @return The mob file name. + */ + public static String getMobFileName(Cell cell) { + return Bytes.toString(cell.getValueArray(), cell.getValueOffset() + Bytes.SIZEOF_INT, + cell.getValueLength() - Bytes.SIZEOF_INT); + } + + /** + * Gets the table name used in the table lock. + * The table lock name is a dummy one, it's not a table name. It's tableName + ".mobLock". + * @param tn The table name. + * @return The table name used in table lock. + */ + public static TableName getTableLockName(TableName tn) { + byte[] tableName = tn.getName(); + return TableName.valueOf(Bytes.add(tableName, MobConstants.MOB_TABLE_LOCK_SUFFIX)); + } + + /** + * Performs the mob file compaction. + * @param conf the Configuration + * @param fs the file system + * @param tableName the table the compact + * @param hcd the column descriptor + * @param pool the thread pool + * @param tableLockManager the tableLock manager + * @param isForceAllFiles Whether add all mob files into the compaction. + */ + public static void doMobFileCompaction(Configuration conf, FileSystem fs, TableName tableName, + HColumnDescriptor hcd, ExecutorService pool, TableLockManager tableLockManager, + boolean isForceAllFiles) throws IOException { + String className = conf.get(MobConstants.MOB_FILE_COMPACTOR_CLASS_KEY, + PartitionedMobFileCompactor.class.getName()); + // instantiate the mob file compactor. + MobFileCompactor compactor = null; + try { + compactor = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { + Configuration.class, FileSystem.class, TableName.class, HColumnDescriptor.class, + ExecutorService.class }, new Object[] { conf, fs, tableName, hcd, pool }); + } catch (Exception e) { + throw new IOException("Unable to load configured mob file compactor '" + className + "'", e); + } + // compact only for mob-enabled column. + // obtain a write table lock before performing compaction to avoid race condition + // with major compaction in mob-enabled column. + boolean tableLocked = false; + TableLock lock = null; + try { + // the tableLockManager might be null in testing. In that case, it is lock-free. + if (tableLockManager != null) { + lock = tableLockManager.writeLock(MobUtils.getTableLockName(tableName), + "Run MobFileCompaction"); + lock.acquire(); + } + tableLocked = true; + compactor.compact(isForceAllFiles); + } catch (Exception e) { + LOG.error("Fail to compact the mob files for the column " + hcd.getNameAsString() + + " in the table " + tableName.getNameAsString(), e); + } finally { + if (lock != null && tableLocked) { + try { + lock.release(); + } catch (IOException e) { + LOG.error("Fail to release the write lock for the table " + tableName.getNameAsString(), + e); + } + } + } + } + + /** + * Creates a thread pool. + * @param conf the Configuration + * @return A thread pool. + */ + public static ExecutorService createMobFileCompactorThreadPool(Configuration conf) { + int maxThreads = conf.getInt(MobConstants.MOB_FILE_COMPACTION_THREADS_MAX, + MobConstants.DEFAULT_MOB_FILE_COMPACTION_THREADS_MAX); + if (maxThreads == 0) { + maxThreads = 1; + } + final SynchronousQueue queue = new SynchronousQueue(); + ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue, + Threads.newDaemonThreadFactory("MobFileCompactor"), new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + try { + // waiting for a thread to pick up instead of throwing exceptions. + queue.put(r); + } catch (InterruptedException e) { + throw new RejectedExecutionException(e); + } + } + }); + ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); + return pool; + } +}