hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject [18/50] [abbrv] hbase git commit: Merge remote-tracking branch 'apache/master' (5/19/15) into hbase-11339
Date Wed, 22 Jul 2015 19:52:36 GMT
Merge remote-tracking branch 'apache/master' (5/19/15) into hbase-11339

Patches that caused deltas:
HBASE-10810 - around HColumnDescriptor 'should' vs 'is' api.
HBASE-11677 - LOG was made private
HBASE-11927 - Checksum constant changed
HBASE-10800 - CellComparator instead of KVComparator

Conflicts:
	hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
	hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09a00efc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09a00efc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09a00efc

Branch: refs/heads/master
Commit: 09a00efc0b37d6a43f684ee30f8b266d27c58c1c
Parents: 8a09d95 74e5b26
Author: Jonathan M Hsieh <jmhsieh@apache.org>
Authored: Tue May 19 16:30:21 2015 -0700
Committer: Jonathan M Hsieh <jmhsieh@apache.org>
Committed: Tue May 19 16:30:21 2015 -0700

----------------------------------------------------------------------
 .gitignore                                      |   1 +
 bin/draining_servers.rb                         |   2 +-
 bin/hbase-daemon.sh                             |   2 +-
 bin/hbase.cmd                                   |  12 +-
 bin/region_mover.rb                             |  69 +-
 bin/region_status.rb                            |   4 +-
 conf/hbase-env.cmd                              |   4 +
 conf/hbase-env.sh                               |   4 +
 dev-support/check_compatibility.sh              |   2 +-
 dev-support/test-patch.properties               |   4 +-
 dev-support/test-patch.sh                       |   2 +-
 hbase-assembly/pom.xml                          |   3 -
 .../src/main/assembly/hadoop-two-compat.xml     |  18 +-
 hbase-assembly/src/main/assembly/src.xml        |  26 +
 .../org/apache/hadoop/hbase/ClusterStatus.java  |  20 -
 .../apache/hadoop/hbase/HColumnDescriptor.java  | 312 ++-----
 .../org/apache/hadoop/hbase/HRegionInfo.java    |  59 +-
 .../apache/hadoop/hbase/HTableDescriptor.java   |   6 +-
 .../org/apache/hadoop/hbase/ServerLoad.java     |   4 +-
 .../org/apache/hadoop/hbase/client/Admin.java   | 160 +++-
 .../hadoop/hbase/client/AsyncProcess.java       |   2 +-
 .../client/ClientAsyncPrefetchScanner.java      | 234 ++++++
 .../hadoop/hbase/client/ClientIdGenerator.java  |   2 +-
 .../hadoop/hbase/client/ClientScanner.java      | 143 +++-
 .../hbase/client/ClientSimpleScanner.java       |  54 ++
 .../hadoop/hbase/client/ClientSmallScanner.java |  17 +-
 .../hbase/client/ConnectionImplementation.java  |   4 +-
 .../apache/hadoop/hbase/client/HBaseAdmin.java  | 785 ++++++++++--------
 .../org/apache/hadoop/hbase/client/HTable.java  |  41 +-
 .../hadoop/hbase/client/HTableInterface.java    |  10 +-
 .../apache/hadoop/hbase/client/MetaCache.java   |   7 +-
 .../client/PreemptiveFastFailInterceptor.java   |   2 +-
 .../hbase/client/RegionServerCallable.java      |   2 +-
 .../org/apache/hadoop/hbase/client/Result.java  |  13 +-
 .../hadoop/hbase/client/ResultScanner.java      |   6 +
 .../hbase/client/ReversedClientScanner.java     |   2 +-
 .../hbase/client/RpcRetryingCallerImpl.java     |   1 +
 .../RpcRetryingCallerWithReadReplicas.java      |   2 +-
 .../org/apache/hadoop/hbase/client/Scan.java    |  30 +-
 .../hadoop/hbase/client/ScannerCallable.java    |  26 +-
 .../client/ScannerCallableWithReplicas.java     |  12 +-
 .../hadoop/hbase/client/TableConfiguration.java |  24 +-
 .../client/UnmodifyableHTableDescriptor.java    |   7 +-
 .../hadoop/hbase/client/ZooKeeperRegistry.java  |   2 +-
 .../apache/hadoop/hbase/filter/FilterList.java  |   6 +-
 .../hadoop/hbase/filter/FuzzyRowFilter.java     |   5 +-
 .../hbase/filter/MultiRowRangeFilter.java       |  10 +-
 .../apache/hadoop/hbase/filter/ParseFilter.java |   6 +-
 .../hbase/filter/SingleColumnValueFilter.java   |   2 +-
 .../hadoop/hbase/ipc/AbstractRpcClient.java     |   3 +-
 .../org/apache/hadoop/hbase/ipc/AsyncCall.java  |   4 +-
 .../hadoop/hbase/ipc/AsyncRpcChannel.java       |   2 +-
 .../apache/hadoop/hbase/ipc/AsyncRpcClient.java |   7 +-
 .../hbase/ipc/AsyncServerResponseHandler.java   |   4 +-
 .../hadoop/hbase/ipc/CoprocessorRpcChannel.java |   2 +-
 .../org/apache/hadoop/hbase/ipc/IPCUtil.java    |   1 +
 .../hbase/ipc/MasterCoprocessorRpcChannel.java  |   2 +-
 .../hbase/ipc/RegionCoprocessorRpcChannel.java  |   2 +-
 .../ipc/RegionServerCoprocessorRpcChannel.java  |   2 +-
 .../apache/hadoop/hbase/ipc/RpcClientImpl.java  |   4 +
 .../hadoop/hbase/protobuf/ProtobufUtil.java     |   2 +-
 .../hadoop/hbase/protobuf/RequestConverter.java |   3 +
 .../hbase/protobuf/ResponseConverter.java       |  22 +-
 .../hbase/security/HBaseSaslRpcClient.java      |   2 +-
 .../hbase/security/SaslClientHandler.java       |   4 +-
 .../hbase/security/SecureBulkLoadUtil.java      |   2 +-
 .../hbase/security/access/TablePermission.java  |   2 +-
 .../hbase/security/access/UserPermission.java   |   2 +-
 .../token/AuthenticationTokenSelector.java      |   2 +-
 .../hadoop/hbase/security/token/TokenUtil.java  |   2 +-
 .../security/visibility/VisibilityClient.java   | 212 +++--
 .../hbase/zookeeper/MasterAddressTracker.java   |  30 +-
 .../hbase/zookeeper/RecoverableZooKeeper.java   |   6 +-
 .../hadoop/hbase/zookeeper/ZKLeaderManager.java |   2 +-
 .../hbase/zookeeper/ZooKeeperNodeTracker.java   |   3 +-
 .../hadoop/hbase/TestHTableDescriptor.java      |   2 +-
 .../hbase/TestInterfaceAudienceAnnotations.java |  27 +-
 .../hadoop/hbase/client/TestAsyncProcess.java   |   3 +
 .../hbase/client/TestClientNoCluster.java       |   4 +-
 .../hadoop/hbase/client/TestClientScanner.java  |  10 +
 .../client/TestClientSmallReversedScanner.java  |  32 +-
 .../hbase/client/TestClientSmallScanner.java    |  34 +-
 .../hadoop/hbase/client/TestOperation.java      |   9 +-
 .../hbase/client/TestProcedureFuture.java       |  17 +-
 .../apache/hadoop/hbase/ipc/TestIPCUtil.java    |   2 +-
 .../java/org/apache/hadoop/hbase/AuthUtil.java  |   2 +-
 .../org/apache/hadoop/hbase/CellComparator.java | 827 ++++++++++++-------
 .../java/org/apache/hadoop/hbase/CellUtil.java  | 146 +++-
 .../org/apache/hadoop/hbase/ChoreService.java   |   8 +-
 .../org/apache/hadoop/hbase/HConstants.java     |  17 +-
 .../java/org/apache/hadoop/hbase/KeyValue.java  |  50 +-
 .../apache/hadoop/hbase/KeyValueTestUtil.java   |   2 +-
 .../org/apache/hadoop/hbase/KeyValueUtil.java   | 123 ++-
 .../org/apache/hadoop/hbase/NoTagsKeyValue.java |  37 +
 .../org/apache/hadoop/hbase/ScheduledChore.java |   8 +-
 .../java/org/apache/hadoop/hbase/TableName.java |   6 +-
 .../hadoop/hbase/codec/KeyValueCodec.java       |   3 +-
 .../hbase/codec/KeyValueCodecWithTags.java      |   3 +-
 .../hadoop/hbase/io/BoundedByteBufferPool.java  |   4 +-
 .../hadoop/hbase/io/compress/Compression.java   |   2 +-
 .../io/encoding/BufferedDataBlockEncoder.java   | 162 ++--
 .../io/encoding/CopyKeyDataBlockEncoder.java    |   4 +-
 .../hbase/io/encoding/DataBlockEncoder.java     |  10 +-
 .../hbase/io/encoding/DiffKeyDeltaEncoder.java  |   4 +-
 .../hbase/io/encoding/FastDiffDeltaEncoder.java |   4 +-
 .../io/encoding/PrefixKeyDeltaEncoder.java      |   4 +-
 .../hadoop/hbase/io/hfile/HFileContext.java     |   3 +-
 .../hbase/io/hfile/HFileContextBuilder.java     |   3 +-
 .../hadoop/hbase/util/ByteBufferArray.java      |   2 +-
 .../org/apache/hadoop/hbase/util/Bytes.java     | 222 ++++-
 .../apache/hadoop/hbase/util/ChecksumType.java  |  95 +--
 .../org/apache/hadoop/hbase/util/ClassSize.java |   2 +-
 .../java/org/apache/hadoop/hbase/util/JVM.java  |   9 +
 .../org/apache/hadoop/hbase/util/Methods.java   |   2 +-
 .../org/apache/hadoop/hbase/util/Sleeper.java   |   2 +-
 .../org/apache/hadoop/hbase/util/Threads.java   |   2 +-
 .../hbase/util/test/RedundantKVGenerator.java   |   3 +-
 .../src/main/resources/hbase-default.xml        |  28 +-
 .../apache/hadoop/hbase/TestCellComparator.java | 100 +--
 .../apache/hadoop/hbase/TestChoreService.java   | 764 +++++++++--------
 .../org/apache/hadoop/hbase/TestKeyValue.java   | 147 +---
 .../hadoop/hbase/codec/TestCellCodec.java       |   7 +-
 .../hbase/codec/TestCellCodecWithTags.java      |   8 +-
 .../hbase/codec/TestKeyValueCodecWithTags.java  |   8 +-
 .../io/crypto/TestKeyStoreKeyProvider.java      |   2 +-
 .../org/apache/hadoop/hbase/util/TestBytes.java |  41 +-
 .../apache/hadoop/hbase/types/TestPBCell.java   |   3 +-
 .../hbase/ipc/MetricsHBaseServerSource.java     |  22 +
 .../hbase/master/MetricsMasterSource.java       |   4 +-
 .../MetricsHBaseServerSourceFactoryImpl.java    |   6 +-
 .../hbase/ipc/MetricsHBaseServerSourceImpl.java |  71 +-
 .../hbase/master/MetricsMasterSourceImpl.java   |   3 +-
 .../hadoop/hbase/IntegrationTestIngest.java     |   1 +
 .../IntegrationTestIngestWithEncryption.java    |   6 +-
 ...tegrationTestIngestWithVisibilityLabels.java |   4 +-
 .../hbase/IntegrationTestMetaReplicas.java      |   2 +-
 .../hadoop/hbase/chaos/actions/Action.java      |   2 +-
 .../actions/RollingBatchRestartRsAction.java    |   2 +-
 .../hadoop/hbase/chaos/policies/Policy.java     |   2 +-
 .../mapreduce/IntegrationTestImportTsv.java     |   7 +-
 .../test/IntegrationTestBigLinkedList.java      |   2 +-
 ...egrationTestBigLinkedListWithVisibility.java |  21 +-
 ...tionTestWithCellVisibilityLoadAndVerify.java |  16 +-
 .../hbase/codec/prefixtree/PrefixTreeCodec.java |  18 +-
 .../codec/prefixtree/PrefixTreeSeeker.java      |  12 +-
 .../decode/PrefixTreeArrayScanner.java          |   4 +-
 .../codec/prefixtree/decode/PrefixTreeCell.java |  25 +-
 .../codec/prefixtree/row/BaseTestRowData.java   |   4 +-
 .../prefixtree/row/TestPrefixTreeSearcher.java  |   6 +-
 .../row/data/TestRowDataNumberStrings.java      |   2 +-
 .../row/data/TestRowDataSearcherRowMiss.java    |  14 +-
 .../prefixtree/row/data/TestRowDataSimple.java  |  14 +-
 .../hbase/procedure2/OnePhaseProcedure.java     |   2 +-
 .../procedure2/ProcedureAbortedException.java   |   2 +-
 .../hbase/procedure2/ProcedureException.java    |   2 +-
 .../hbase/procedure2/ProcedureExecutor.java     |  68 +-
 .../hbase/procedure2/ProcedureResult.java       |   2 +-
 .../procedure2/ProcedureYieldException.java     |   2 +-
 .../procedure2/RemoteProcedureException.java    |   2 +-
 .../hbase/procedure2/SequentialProcedure.java   |   2 +-
 .../hbase/procedure2/StateMachineProcedure.java |   2 +-
 .../hbase/procedure2/TwoPhaseProcedure.java     |   2 +-
 .../hbase/procedure2/store/ProcedureStore.java  |   2 +-
 .../procedure2/store/ProcedureStoreTracker.java |  21 +-
 .../CorruptedWALProcedureStoreException.java    |   2 +-
 .../procedure2/store/wal/WALProcedureStore.java | 134 ++-
 .../hbase/procedure2/util/StringUtils.java      |   4 +
 .../store/TestProcedureStoreTracker.java        |  47 ++
 .../hbase/protobuf/generated/ClientProtos.java  | 349 ++++++--
 .../protobuf/generated/ClusterStatusProtos.java | 100 +--
 .../hbase/protobuf/generated/MasterProtos.java  | 589 ++++++++-----
 .../apache/hadoop/hbase/util/ByteStringer.java  |   2 +-
 hbase-protocol/src/main/protobuf/Client.proto   |   7 +
 .../src/main/protobuf/ClusterStatus.proto       |   4 +-
 hbase-protocol/src/main/protobuf/Master.proto   |   2 +
 .../apache/hadoop/hbase/rest/RESTServer.java    |   1 +
 .../apache/hadoop/hbase/rest/RESTServlet.java   |   2 +-
 .../hadoop/hbase/rest/SchemaResource.java       |   4 +-
 .../hadoop/hbase/rest/client/RemoteHTable.java  |   5 +
 .../rest/model/StorageClusterStatusModel.java   |  30 +-
 .../generated/StorageClusterStatusMessage.java  |  96 +--
 .../protobuf/StorageClusterStatusMessage.proto  |   4 +-
 .../apache/hadoop/hbase/rest/DummyFilter.java   |   2 +-
 .../hbase/rest/HBaseRESTTestingUtility.java     |   2 +-
 .../hbase/rest/TestScannersWithLabels.java      |  10 +-
 .../hadoop/hbase/rest/TestStatusResource.java   |   2 +-
 hbase-server/pom.xml                            |   1 +
 .../tmpl/master/BackupMasterStatusTmpl.jamon    |  51 +-
 .../hbase/tmpl/master/MasterStatusTmpl.jamon    |   5 +-
 .../tmpl/regionserver/RegionListTmpl.jamon      |   6 +-
 .../apache/hadoop/hbase/HealthCheckChore.java   |   2 +-
 .../org/apache/hadoop/hbase/HealthChecker.java  |   2 +-
 .../org/apache/hadoop/hbase/JMXListener.java    |  18 +-
 .../apache/hadoop/hbase/LocalHBaseCluster.java  |   2 +-
 .../apache/hadoop/hbase/SplitLogCounters.java   |   5 +-
 .../org/apache/hadoop/hbase/TagRewriteCell.java |   5 +
 .../apache/hadoop/hbase/ZKNamespaceManager.java |   2 +-
 .../org/apache/hadoop/hbase/ZNodeClearer.java   |   2 +-
 .../hbase/client/ClientSideRegionScanner.java   |   9 +-
 .../hbase/client/TableSnapshotScanner.java      |   5 +
 .../hadoop/hbase/conf/ConfigurationManager.java |   2 +-
 .../ForeignExceptionDispatcher.java             |   2 +-
 .../hadoop/hbase/http/HttpRequestLog.java       |   2 +-
 .../apache/hadoop/hbase/http/HttpServer.java    |  13 +-
 .../hadoop/hbase/io/HalfStoreFileReader.java    |  26 +-
 .../hadoop/hbase/io/hfile/ChecksumUtil.java     | 110 +--
 .../hadoop/hbase/io/hfile/FixedFileTrailer.java |  76 +-
 .../org/apache/hadoop/hbase/io/hfile/HFile.java |  31 +-
 .../hadoop/hbase/io/hfile/HFileBlockIndex.java  |  50 +-
 .../hbase/io/hfile/HFilePrettyPrinter.java      |  14 +-
 .../hadoop/hbase/io/hfile/HFileReaderImpl.java  | 270 ++++--
 .../hbase/io/hfile/HFileWriterFactory.java      |   4 +-
 .../hadoop/hbase/io/hfile/HFileWriterImpl.java  | 141 +++-
 .../hadoop/hbase/io/hfile/LruBlockCache.java    |   2 +-
 .../hbase/io/hfile/bucket/BucketAllocator.java  |   2 +-
 .../hbase/io/hfile/bucket/BucketCache.java      |   2 +-
 .../org/apache/hadoop/hbase/ipc/CallRunner.java |  14 +-
 .../hadoop/hbase/ipc/MetricsHBaseServer.java    |  34 +
 .../apache/hadoop/hbase/ipc/QosPriority.java    |  32 +
 .../hadoop/hbase/ipc/RWQueueRpcExecutor.java    |   3 +
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  |  23 +-
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java    |   2 +-
 .../hbase/mapred/TableRecordReaderImpl.java     |   2 +-
 .../hbase/mapreduce/HFileOutputFormat.java      |   2 +-
 .../hbase/mapreduce/HFileOutputFormat2.java     |   9 +-
 .../hbase/mapreduce/KeyValueSerialization.java  |   5 +-
 .../hbase/mapreduce/KeyValueSortReducer.java    |   3 +-
 .../hbase/mapreduce/LoadIncrementalHFiles.java  |  82 +-
 .../mapreduce/MultiTableInputFormatBase.java    |   2 +-
 .../hadoop/hbase/mapreduce/PutSortReducer.java  |   3 +-
 .../mapreduce/SimpleTotalOrderPartitioner.java  |   2 +-
 .../hbase/mapreduce/TableInputFormatBase.java   |   2 +-
 .../hbase/mapreduce/TableMapReduceUtil.java     |   7 +-
 .../hbase/mapreduce/TableRecordReaderImpl.java  |   2 +-
 .../hadoop/hbase/mapreduce/TableSplit.java      |  24 +-
 .../hadoop/hbase/mapreduce/TextSortReducer.java |   3 +-
 .../hadoop/hbase/mapreduce/WALPlayer.java       |   2 +-
 .../hadoop/hbase/master/AssignmentManager.java  |  90 +-
 .../master/AssignmentVerificationReport.java    |   2 +-
 .../hbase/master/GeneralBulkAssigner.java       |  29 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  26 +-
 .../hadoop/hbase/master/HMasterCommandLine.java |  27 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |  15 +-
 .../hadoop/hbase/master/MasterServices.java     |   4 +-
 .../hadoop/hbase/master/MetricsMaster.java      |   4 +-
 .../apache/hadoop/hbase/master/RackManager.java |   2 +-
 .../hbase/master/balancer/FavoredNodesPlan.java |   2 +-
 .../master/balancer/RegionLocationFinder.java   |   2 +-
 .../hbase/master/cleaner/HFileLinkCleaner.java  |  13 +-
 .../hadoop/hbase/master/cleaner/LogCleaner.java |   2 +-
 .../master/cleaner/TimeToLiveHFileCleaner.java  |   2 +-
 .../master/cleaner/TimeToLiveLogCleaner.java    |   2 +-
 .../master/handler/ModifyTableHandler.java      | 149 ----
 .../master/handler/TableAddFamilyHandler.java   |  89 --
 .../handler/TableDeleteFamilyHandler.java       |  90 --
 .../handler/TableModifyFamilyHandler.java       |  84 --
 .../master/handler/TruncateTableHandler.java    | 142 ----
 .../master/procedure/CreateTableProcedure.java  |   5 +
 .../master/procedure/DeleteTableProcedure.java  |  14 +-
 .../master/procedure/ProcedureSyncWait.java     |  12 +
 .../hbase/master/snapshot/SnapshotManager.java  |  37 +-
 .../org/apache/hadoop/hbase/mob/MobUtils.java   |  23 +-
 .../PartitionedMobFileCompactor.java            |  11 +-
 .../hbase/namespace/NamespaceAuditor.java       |  18 +-
 .../hbase/namespace/NamespaceStateManager.java  |  29 +-
 .../hadoop/hbase/procedure/ProcedureMember.java |  22 +-
 .../hadoop/hbase/procedure/Subprocedure.java    |   9 +-
 .../procedure/ZKProcedureCoordinatorRpcs.java   |   2 +-
 .../hbase/procedure/ZKProcedureMemberRpcs.java  |  18 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   6 +
 .../AnnotationReadingPriorityFunction.java      |  52 +-
 .../hbase/regionserver/CellSkipListSet.java     |   4 +-
 .../hbase/regionserver/CompactSplitThread.java  |   5 +-
 .../hbase/regionserver/DefaultMemStore.java     |   7 +-
 .../hbase/regionserver/DefaultStoreEngine.java  |   4 +-
 .../regionserver/DefaultStoreFileManager.java   |   8 +-
 .../hbase/regionserver/DefaultStoreFlusher.java |   2 +-
 .../GetClosestRowBeforeTracker.java             |  17 +-
 .../hadoop/hbase/regionserver/HMobStore.java    |  20 +-
 .../hadoop/hbase/regionserver/HRegion.java      | 257 ++++--
 .../hbase/regionserver/HRegionFileSystem.java   |  50 +-
 .../hbase/regionserver/HRegionServer.java       |  32 +-
 .../hadoop/hbase/regionserver/HStore.java       |  27 +-
 ...IncreasingToUpperBoundRegionSplitPolicy.java |   2 +-
 .../hadoop/hbase/regionserver/KeyValueHeap.java |  12 +-
 .../hadoop/hbase/regionserver/LogRoller.java    |   2 +-
 .../hadoop/hbase/regionserver/LruHashMap.java   |   2 +-
 .../hbase/regionserver/MemStoreFlusher.java     |   2 +-
 .../MetricsRegionServerWrapperImpl.java         |   2 +-
 .../regionserver/NoLimitScannerContext.java     |  28 +-
 .../hbase/regionserver/RSDumpServlet.java       |  22 +-
 .../hbase/regionserver/RSRpcServices.java       | 189 ++++-
 .../hadoop/hbase/regionserver/Region.java       |   4 +
 .../regionserver/RegionCoprocessorHost.java     |   8 +-
 .../hbase/regionserver/RegionMergeRequest.java  |   2 +-
 .../hbase/regionserver/RegionSplitPolicy.java   |  11 +
 .../regionserver/ReversedKeyValueHeap.java      |  10 +-
 .../regionserver/ReversedRegionScannerImpl.java |   6 +-
 .../regionserver/ReversedStoreScanner.java      |   9 +-
 .../hadoop/hbase/regionserver/ScanInfo.java     |  10 +-
 .../hbase/regionserver/ScanQueryMatcher.java    |  45 +-
 .../hbase/regionserver/ScannerContext.java      | 128 ++-
 .../hadoop/hbase/regionserver/SplitRequest.java |   2 +-
 .../apache/hadoop/hbase/regionserver/Store.java |   4 +-
 .../hadoop/hbase/regionserver/StoreEngine.java  |   8 +-
 .../hadoop/hbase/regionserver/StoreFile.java    | 137 +--
 .../hbase/regionserver/StoreFileInfo.java       |   7 +-
 .../hbase/regionserver/StoreFileScanner.java    |  22 +-
 .../hadoop/hbase/regionserver/StoreScanner.java |  41 +-
 .../regionserver/StripeMultiFileWriter.java     |  60 +-
 .../hbase/regionserver/StripeStoreConfig.java   |   2 +-
 .../hbase/regionserver/StripeStoreEngine.java   |   6 +-
 .../regionserver/StripeStoreFileManager.java    |  47 +-
 .../hbase/regionserver/StripeStoreFlusher.java  |   2 +-
 .../compactions/CompactionConfiguration.java    |   2 +-
 .../compactions/CompactionRequest.java          |   2 +-
 .../regionserver/compactions/Compactor.java     |   2 +-
 .../compactions/DefaultCompactor.java           |  11 +-
 .../compactions/RatioBasedCompactionPolicy.java |   4 +-
 .../compactions/StripeCompactionPolicy.java     |   2 +-
 .../compactions/StripeCompactor.java            |   6 +-
 .../hadoop/hbase/regionserver/wal/FSHLog.java   |   8 +-
 .../hadoop/hbase/regionserver/wal/HLogKey.java  |   2 +-
 .../hbase/regionserver/wal/MetricsWAL.java      |   2 +-
 .../regionserver/wal/MetricsWALEditsReplay.java |   2 +-
 .../regionserver/wal/ProtobufLogWriter.java     |   2 +-
 .../hbase/regionserver/wal/WALCellCodec.java    |   3 +-
 .../hadoop/hbase/regionserver/wal/WALEdit.java  |  11 +-
 .../hadoop/hbase/regionserver/wal/WALUtil.java  |   2 +-
 .../HBaseInterClusterReplicationEndpoint.java   |   1 +
 .../replication/regionserver/MetricsSource.java |   2 +-
 .../regionserver/ReplicationSinkManager.java    |  10 +
 .../regionserver/ReplicationSource.java         |   2 +-
 .../regionserver/ReplicationSyncUp.java         |   2 +-
 .../hbase/security/HBaseSaslRpcServer.java      |   2 +-
 .../security/access/AccessControlLists.java     |   2 +-
 .../hbase/security/access/AccessController.java |   4 +-
 .../HbaseObjectWritableFor96Migration.java      |   2 +-
 .../security/access/SecureBulkLoadEndpoint.java |   2 +-
 .../hbase/security/access/TableAuthManager.java |   2 +-
 .../security/access/ZKPermissionWatcher.java    |   2 +-
 .../token/AuthenticationTokenSecretManager.java |   2 +-
 .../hbase/security/token/TokenProvider.java     |   2 +-
 .../hbase/security/token/ZKSecretWatcher.java   |   2 +-
 .../visibility/VisibilityController.java        |   3 +-
 .../hbase/snapshot/SnapshotReferenceUtil.java   |   2 +-
 .../apache/hadoop/hbase/util/BloomFilter.java   |  61 +-
 .../hadoop/hbase/util/BloomFilterBase.java      |  13 -
 .../hadoop/hbase/util/BloomFilterChunk.java     | 322 ++++++++
 .../hadoop/hbase/util/BloomFilterFactory.java   |  12 +-
 .../hadoop/hbase/util/BloomFilterUtil.java      | 269 ++++++
 .../hadoop/hbase/util/BloomFilterWriter.java    |   4 -
 .../util/BoundedConcurrentLinkedQueue.java      | 122 +++
 .../hadoop/hbase/util/ByteBloomFilter.java      | 662 ---------------
 .../hbase/util/CollectionBackedScanner.java     |  14 +-
 .../hadoop/hbase/util/CompoundBloomFilter.java  |  54 +-
 .../hbase/util/CompoundBloomFilterBase.java     |  33 +-
 .../hbase/util/CompoundBloomFilterWriter.java   |  39 +-
 .../hadoop/hbase/util/CompressionTest.java      |   4 +-
 .../hadoop/hbase/util/ConnectionCache.java      |   3 +-
 .../hadoop/hbase/util/EncryptionTest.java       |   2 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java   |  61 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  85 +-
 .../hadoop/hbase/util/HBaseFsckRepair.java      |   8 +-
 .../org/apache/hadoop/hbase/util/HMerge.java    |  11 +-
 .../org/apache/hadoop/hbase/util/Merge.java     |   2 +-
 .../hadoop/hbase/util/RegionSizeCalculator.java |   9 +-
 .../hbase/util/RegionSplitCalculator.java       |   3 +-
 .../hadoop/hbase/util/RegionSplitter.java       |   4 +-
 .../org/apache/hadoop/hbase/wal/WALFactory.java |  11 +-
 .../org/apache/hadoop/hbase/wal/WALKey.java     |   2 +-
 .../apache/hadoop/hbase/wal/WALSplitter.java    |   2 +-
 .../native/src/mlockall_agent/mlockall_agent.c  |  64 +-
 .../hbase-webapps/master/tablesDetailed.jsp     |  13 +-
 .../org/apache/hadoop/hbase/HBaseCluster.java   |   1 +
 .../hadoop/hbase/HBaseTestingUtility.java       |   2 +-
 .../hbase/HFilePerformanceEvaluation.java       |   4 +-
 .../hbase/MapFilePerformanceEvaluation.java     |   2 +-
 .../apache/hadoop/hbase/MetaMockingUtil.java    |   2 +-
 .../apache/hadoop/hbase/MiniHBaseCluster.java   |   2 +-
 .../hadoop/hbase/MultithreadedTestUtil.java     |   2 +-
 .../hadoop/hbase/PerformanceEvaluation.java     |   2 +-
 .../hbase/PerformanceEvaluationCommons.java     |   2 +-
 .../hadoop/hbase/TestGlobalMemStoreSize.java    |   2 +-
 .../hadoop/hbase/TestHBaseTestingUtility.java   |   2 +-
 .../TestHColumnDescriptorDefaultVersions.java   |   8 +-
 .../org/apache/hadoop/hbase/TestIOFencing.java  |   2 +-
 .../apache/hadoop/hbase/TestInfoServers.java    |   2 +-
 .../hadoop/hbase/TestMovedRegionsCleaner.java   |   2 +-
 .../org/apache/hadoop/hbase/TestNamespace.java  |   2 +-
 .../hbase/TestPartialResultsFromClientSide.java |   3 +-
 .../hadoop/hbase/TestRegionRebalancing.java     |   2 +-
 .../apache/hadoop/hbase/TestSerialization.java  |   8 +-
 .../hadoop/hbase/TestTableDescriptor.java       |   2 +-
 .../apache/hadoop/hbase/TestTagRewriteCell.java |  48 ++
 .../org/apache/hadoop/hbase/TestZooKeeper.java  |   2 +-
 .../hadoop/hbase/backup/TestHFileArchiving.java |   2 +-
 .../apache/hadoop/hbase/client/TestAdmin1.java  |  32 +-
 .../apache/hadoop/hbase/client/TestAdmin2.java  |   3 +-
 .../client/TestClientScannerRPCTimeout.java     |   2 +-
 .../hadoop/hbase/client/TestClientTimeouts.java |   2 +-
 .../client/TestCloneSnapshotFromClient.java     |   3 +-
 .../hadoop/hbase/client/TestFastFail.java       |   9 +-
 .../hadoop/hbase/client/TestFromClientSide.java |  39 +-
 .../hbase/client/TestFromClientSide3.java       |  28 +-
 .../hbase/client/TestHTableMultiplexer.java     |   2 +-
 .../client/TestHTableMultiplexerFlushCache.java |   2 +-
 .../hbase/client/TestMetaWithReplicas.java      |   2 +-
 .../hbase/client/TestMultipleTimestamps.java    |   2 +-
 .../hbase/client/TestReplicaWithCluster.java    |   2 +-
 .../client/TestRestoreSnapshotFromClient.java   |   4 +-
 .../apache/hadoop/hbase/client/TestResult.java  |  17 +-
 .../hadoop/hbase/client/TestScannerTimeout.java |   2 +-
 .../client/TestScannersFromClientSide.java      |  99 +++
 .../hadoop/hbase/client/TestSizeFailures.java   |   2 +-
 .../client/TestSnapshotCloneIndependence.java   |  73 +-
 .../hbase/client/TestSnapshotMetadata.java      |   2 +-
 .../hbase/client/TestTimestampsFilter.java      |   2 +-
 .../replication/TestReplicationAdmin.java       |  16 +-
 .../hadoop/hbase/codec/CodecPerformance.java    |   2 +
 .../hbase/codec/TestCellMessageCodec.java       |   9 +-
 .../hbase/conf/TestConfigurationManager.java    |   2 +-
 .../coprocessor/ColumnAggregationEndpoint.java  |   2 +-
 .../ColumnAggregationEndpointNullResponse.java  |   2 +-
 .../ColumnAggregationEndpointWithErrors.java    |   2 +-
 .../coprocessor/SampleRegionWALObserver.java    |   2 +-
 .../hbase/coprocessor/SimpleRegionObserver.java |   2 +-
 .../coprocessor/TestCoprocessorEndpoint.java    |   1 +
 .../coprocessor/TestCoprocessorInterface.java   |   3 +-
 .../hbase/coprocessor/TestMasterObserver.java   |  10 +-
 .../TestRegionObserverInterface.java            |   2 +-
 .../TestRegionServerCoprocessorEndpoint.java    |  38 +-
 ...gionServerCoprocessorExceptionWithAbort.java |   5 +-
 .../coprocessor/TestRowProcessorEndpoint.java   |   2 +-
 .../DummyRegionServerEndpointProtos.java        |  81 +-
 .../hbase/filter/FilterTestingCluster.java      |  10 +-
 .../hbase/filter/TestColumnRangeFilter.java     |   2 +-
 .../hbase/filter/TestDependentColumnFilter.java |   5 +-
 .../apache/hadoop/hbase/filter/TestFilter.java  |   9 +-
 .../hadoop/hbase/filter/TestFilterList.java     |  15 +-
 .../TestFuzzyRowAndColumnRangeFilter.java       |   2 +-
 .../hadoop/hbase/filter/TestFuzzyRowFilter.java |   6 +-
 .../hbase/filter/TestMultiRowRangeFilter.java   |   4 +-
 .../TestSingleColumnValueExcludeFilter.java     |   5 +-
 .../hadoop/hbase/http/TestGlobalFilter.java     |   2 +-
 .../hadoop/hbase/http/TestHttpServer.java       |   2 +-
 .../hadoop/hbase/http/TestPathFilter.java       |   2 +-
 .../hadoop/hbase/http/TestServletFilter.java    |   2 +-
 .../hbase/http/resource/JerseyResource.java     |   2 +-
 .../hbase/io/TestHalfStoreFileReader.java       |   7 +-
 .../apache/hadoop/hbase/io/TestHeapSize.java    |   2 +-
 .../encoding/TestBufferedDataBlockEncoder.java  |  33 +
 .../hbase/io/encoding/TestChangingEncoding.java |   2 +-
 .../io/encoding/TestDataBlockEncoders.java      |   5 +-
 .../encoding/TestLoadAndSwitchEncodeOnDisk.java |   2 +-
 .../io/encoding/TestPrefixTreeEncoding.java     |  15 +-
 .../encoding/TestSeekToBlockWithEncoders.java   |   3 +-
 .../hadoop/hbase/io/hfile/TestCacheOnWrite.java |   3 +-
 .../hadoop/hbase/io/hfile/TestChecksum.java     |  83 +-
 .../hbase/io/hfile/TestFixedFileTrailer.java    |  25 +-
 .../apache/hadoop/hbase/io/hfile/TestHFile.java | 158 +++-
 .../hadoop/hbase/io/hfile/TestHFileBlock.java   |   6 +-
 .../hbase/io/hfile/TestHFileBlockIndex.java     |  24 +-
 .../hadoop/hbase/io/hfile/TestHFileSeek.java    |   6 +-
 .../hbase/io/hfile/TestHFileWriterV2.java       |   6 +-
 .../hbase/io/hfile/TestHFileWriterV3.java       |   9 +-
 .../hadoop/hbase/io/hfile/TestPrefetch.java     |   3 +-
 .../hadoop/hbase/io/hfile/TestReseekTo.java     |   3 +-
 .../hadoop/hbase/io/hfile/TestSeekTo.java       |   3 +-
 .../apache/hadoop/hbase/ipc/TestAsyncIPC.java   |   2 +-
 .../hbase/ipc/TestRpcHandlerException.java      |   2 +-
 .../apache/hadoop/hbase/ipc/TestRpcMetrics.java |  18 +
 .../hbase/ipc/TestSimpleRpcScheduler.java       |   2 +-
 .../hbase/mapreduce/TestHFileOutputFormat.java  |  15 +-
 .../hbase/mapreduce/TestHFileOutputFormat2.java |   9 +-
 .../hbase/mapreduce/TestImportExport.java       |   9 +-
 .../TestImportTSVWithOperationAttributes.java   |   2 +-
 .../TestImportTSVWithVisibilityLabels.java      |   8 +-
 .../hadoop/hbase/mapreduce/TestImportTsv.java   |   2 +-
 .../TestLoadIncrementalHFilesSplitRecovery.java |   4 +-
 .../mapreduce/TestMultiTableInputFormat.java    |   2 +-
 .../hadoop/hbase/mapreduce/TestRowCounter.java  |   2 +-
 .../mapreduce/TestTableInputFormatScanBase.java |   2 +-
 .../hbase/mapreduce/TestWALRecordReader.java    |   2 +-
 .../hadoop/hbase/master/TestCatalogJanitor.java |  11 +-
 .../master/TestDistributedLogSplitting.java     |   3 +-
 .../TestMasterOperationsForRegionReplicas.java  |   2 +-
 .../hbase/master/TestMasterPriorityRpc.java     |  94 +++
 .../hadoop/hbase/master/TestMasterShutdown.java |   4 +-
 .../hbase/master/TestRegionPlacement.java       |   2 +-
 .../hbase/master/TestRegionPlacement2.java      |   2 +-
 .../hadoop/hbase/master/TestRollingRestart.java |   2 +
 .../hbase/master/TestSplitLogManager.java       |   8 +-
 .../hbase/master/TestTableLockManager.java      |   2 +-
 .../hadoop/hbase/master/TestWarmupRegion.java   |   2 +-
 .../handler/TestTableDeleteFamilyHandler.java   |   6 +-
 .../TestTableDescriptorModification.java        |  16 +-
 .../TestPartitionedMobFileCompactor.java        |   9 +-
 .../hbase/namespace/TestNamespaceAuditor.java   | 193 +++++
 .../hbase/procedure/TestProcedureManager.java   |   2 +-
 .../procedure/TestZKProcedureControllers.java   |   2 +-
 .../hadoop/hbase/quotas/TestQuotaAdmin.java     |   4 +-
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |   4 +-
 .../hbase/regionserver/KeyValueScanFixture.java |  12 +-
 .../hbase/regionserver/MockStoreFile.java       |   4 +-
 .../hbase/regionserver/TestAtomicOperation.java |   2 +-
 .../hbase/regionserver/TestBlocksRead.java      |   2 +-
 .../hbase/regionserver/TestCellSkipListSet.java |   3 +-
 .../hbase/regionserver/TestColumnSeeking.java   |   2 +-
 .../hbase/regionserver/TestCompaction.java      |   2 +-
 .../hbase/regionserver/TestCompactionState.java |   2 +-
 .../regionserver/TestCompoundBloomFilter.java   |  32 +-
 .../TestCorruptedRegionStoreFile.java           | 250 ++++++
 .../hbase/regionserver/TestDefaultMemStore.java |  16 +-
 .../regionserver/TestDefaultStoreEngine.java    |   4 +-
 .../regionserver/TestEncryptionKeyRotation.java |   2 +-
 .../regionserver/TestFlushRegionEntry.java      |  37 +-
 .../regionserver/TestGetClosestAtOrBefore.java  |   4 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |  14 +-
 .../regionserver/TestHRegionReplayEvents.java   |  82 +-
 .../regionserver/TestHRegionServerBulkLoad.java |   2 +-
 .../hbase/regionserver/TestJoinedScanners.java  |   2 +-
 .../hbase/regionserver/TestKeyValueHeap.java    |   9 +-
 .../regionserver/TestKeyValueScanFixture.java   |   3 +-
 .../hbase/regionserver/TestMajorCompaction.java |   2 +-
 .../regionserver/TestMasterAddressTracker.java  | 107 ++-
 .../hbase/regionserver/TestMinorCompaction.java |   2 +-
 .../hbase/regionserver/TestMobCompaction.java   |  10 +-
 .../regionserver/TestMultiColumnScanner.java    |   4 +-
 .../hbase/regionserver/TestParallelPut.java     |   2 +-
 .../hbase/regionserver/TestQosFunction.java     |  54 +-
 .../hbase/regionserver/TestQueryMatcher.java    |   6 +-
 .../hbase/regionserver/TestRecoveredEdits.java  |   6 +-
 .../regionserver/TestRegionFavoredNodes.java    |   5 +-
 .../regionserver/TestRegionReplicaFailover.java |   1 +
 .../regionserver/TestRegionServerHostname.java  |   8 +-
 .../TestRegionServerReportForDuty.java          |  18 +-
 .../regionserver/TestRegionSplitPolicy.java     |   4 +-
 .../regionserver/TestReversibleScanners.java    |   5 +-
 .../hadoop/hbase/regionserver/TestScanner.java  |   2 +-
 .../TestScannerHeartbeatMessages.java           | 539 ++++++++++++
 .../TestScannerRetriableFailure.java            | 171 ++++
 .../regionserver/TestSeekOptimizations.java     |   5 +-
 .../TestSplitTransactionOnCluster.java          |  22 +-
 .../hadoop/hbase/regionserver/TestStore.java    |  16 +-
 .../hbase/regionserver/TestStoreFile.java       |  30 +-
 .../hbase/regionserver/TestStoreScanner.java    |  13 +-
 .../hbase/regionserver/TestStripeCompactor.java |   6 +-
 .../regionserver/TestStripeStoreEngine.java     |   4 +-
 .../TestStripeStoreFileManager.java             |   5 +-
 .../hadoop/hbase/regionserver/TestTags.java     |   2 +-
 .../hbase/regionserver/TestWideScanner.java     |   2 +-
 .../compactions/TestStripeCompactionPolicy.java |  15 +-
 .../regionserver/wal/SequenceFileLogWriter.java |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java      |   2 +-
 .../hbase/regionserver/wal/TestProtobufLog.java |   2 +-
 .../wal/TestWALActionsListener.java             |   4 +-
 .../hbase/regionserver/wal/TestWALReplay.java   |   2 +-
 .../replication/TestMasterReplication.java      |  13 +-
 .../replication/TestReplicationEndpoint.java    |   2 +-
 .../TestReplicationWALEntryFilters.java         |   3 +-
 .../TestReplicationSinkManager.java             |  30 +-
 .../TestReplicationSourceManager.java           |   2 +-
 .../regionserver/TestReplicationThrottler.java  |  27 +-
 .../apache/hadoop/hbase/security/TestUser.java  |   2 +-
 .../security/access/TestAccessController.java   |   7 +-
 .../security/token/TestTokenAuthentication.java |   4 +-
 .../security/token/TestZKSecretWatcher.java     |   2 +-
 .../TestDefaultScanLabelGeneratorStack.java     |   6 +-
 .../TestEnforcingScanLabelGenerator.java        |   6 +-
 .../visibility/TestVisibilityLabels.java        |  36 +-
 ...sibilityLabelsOpWithDifferentUsersNoACL.java |  23 +-
 .../TestVisibilityLabelsReplication.java        |  10 +-
 .../visibility/TestVisibilityLabelsWithACL.java |  43 +-
 ...ibilityLabelsWithDefaultVisLabelService.java |  21 +-
 .../TestVisibilityLabelsWithDeletes.java        |  18 +-
 .../TestVisibilityLabelsWithSLGStack.java       |   6 +-
 .../TestVisibilityLablesWithGroups.java         |  19 +-
 .../TestVisibilityWithCheckAuths.java           |  12 +-
 .../TestWithDisabledAuthorization.java          |  27 +-
 .../hbase/snapshot/TestExportSnapshot.java      |   3 +-
 .../TestRestoreFlushSnapshotFromClient.java     |   2 +-
 .../snapshot/TestRestoreSnapshotHelper.java     |   2 +-
 .../apache/hadoop/hbase/util/LoadTestTool.java  |  88 +-
 .../apache/hadoop/hbase/util/MockServer.java    |   2 +-
 .../hadoop/hbase/util/TestBloomFilterChunk.java | 185 +++++
 .../util/TestBoundedConcurrentLinkedQueue.java  | 161 ++++
 .../hadoop/hbase/util/TestByteBloomFilter.java  | 169 ----
 .../hadoop/hbase/util/TestCompressionTest.java  |   2 +-
 .../hadoop/hbase/util/TestConnectionCache.java  |  54 ++
 .../hbase/util/TestCoprocessorScanPolicy.java   |   2 +-
 .../apache/hadoop/hbase/util/TestFSUtils.java   |  45 +-
 .../apache/hadoop/hbase/util/TestFSVisitor.java |   2 +-
 .../apache/hadoop/hbase/util/TestHBaseFsck.java |  15 +-
 .../apache/hadoop/hbase/util/TestMergeTool.java |   2 +-
 .../util/hbck/OfflineMetaRebuildTestCore.java   |   2 +-
 .../util/hbck/TestOfflineMetaRebuildBase.java   |   5 +-
 .../util/hbck/TestOfflineMetaRebuildHole.java   |   5 +-
 .../hbck/TestOfflineMetaRebuildOverlap.java     |   5 +-
 .../hbase/wal/TestDefaultWALProvider.java       |   2 +-
 .../apache/hadoop/hbase/wal/TestSecureWAL.java  |   2 +-
 .../apache/hadoop/hbase/wal/TestWALFactory.java |   8 +-
 .../hbase/wal/TestWALReaderOnSecureWAL.java     |   2 +-
 .../hbase/wal/WALPerformanceEvaluation.java     |   2 +-
 .../hbase/zookeeper/TestZKLeaderManager.java    |   2 +-
 .../protobuf/DummyRegionServerEndpoint.proto    |   1 +
 hbase-shaded/hbase-shaded-client/pom.xml        |  61 ++
 hbase-shaded/hbase-shaded-server/pom.xml        |  61 ++
 hbase-shaded/pom.xml                            | 121 +++
 hbase-shell/src/main/ruby/hbase/admin.rb        |  15 +-
 .../src/main/ruby/hbase/visibility_labels.rb    |  12 +-
 .../src/main/ruby/shell/commands/create.rb      |   6 +-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   |  16 +
 .../apache/hadoop/hbase/thrift/CallQueue.java   |   2 +-
 .../hbase/thrift/HbaseHandlerMetricsProxy.java  |   2 +-
 .../hadoop/hbase/thrift/IncrementCoalescer.java |   2 +-
 .../hadoop/hbase/thrift/ThriftHttpServlet.java  |   2 +-
 .../hadoop/hbase/thrift/ThriftServerRunner.java |   2 +-
 .../hadoop/hbase/thrift/ThriftUtilities.java    |   2 +-
 .../hadoop/hbase/thrift/TestCallQueue.java      |   2 +-
 .../hbase/thrift/TestThriftHttpServer.java      |   2 +-
 .../hbase/thrift/TestThriftServerCmdLine.java   |   2 +-
 .../thrift2/TestThriftHBaseServiceHandler.java  |   2 +-
 ...TestThriftHBaseServiceHandlerWithLabels.java |  10 +-
 pom.xml                                         |  27 +-
 .../asciidoc/_chapters/appendix_acl_matrix.adoc |  94 +--
 src/main/asciidoc/_chapters/architecture.adoc   |  14 +
 src/main/asciidoc/_chapters/configuration.adoc  |  38 +-
 src/main/asciidoc/_chapters/datamodel.adoc      |   2 +-
 src/main/asciidoc/_chapters/developer.adoc      |  20 +-
 .../asciidoc/_chapters/getting_started.adoc     |  10 +-
 src/main/asciidoc/_chapters/hbase_apis.adoc     | 109 ++-
 src/main/asciidoc/_chapters/mapreduce.adoc      |  27 +-
 src/main/asciidoc/_chapters/ops_mgt.adoc        | 187 ++++-
 src/main/asciidoc/_chapters/schema_design.adoc  |  96 ++-
 src/main/asciidoc/_chapters/upgrading.adoc      |   8 +-
 636 files changed, 11702 insertions(+), 6145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 17c1ee3,6790c7e..4b8d6b8
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@@ -41,9 -40,9 +41,10 @@@ import org.apache.hadoop.hbase.classifi
  import org.apache.hadoop.hbase.client.TableState;
  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
  import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
+ import org.apache.hadoop.hbase.ipc.QosPriority;
  import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
  import org.apache.hadoop.hbase.ipc.ServerRpcController;
 +import org.apache.hadoop.hbase.mob.MobUtils;
  import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
  import org.apache.hadoop.hbase.procedure2.Procedure;
  import org.apache.hadoop.hbase.procedure2.ProcedureResult;

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hbase/blob/09a00efc/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 2159597,0000000..9f08c6e
mode 100644,000000..100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@@ -1,751 -1,0 +1,742 @@@
 +/**
 + *
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.hadoop.hbase.mob;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.text.ParseException;
 +import java.text.SimpleDateFormat;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.UUID;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.RejectedExecutionException;
 +import java.util.concurrent.RejectedExecutionHandler;
 +import java.util.concurrent.SynchronousQueue;
 +import java.util.concurrent.ThreadPoolExecutor;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.commons.logging.Log;
 +import org.apache.commons.logging.LogFactory;
 +import org.apache.hadoop.classification.InterfaceAudience;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.hbase.Cell;
- import org.apache.hadoop.hbase.HColumnDescriptor;
- import org.apache.hadoop.hbase.HConstants;
- import org.apache.hadoop.hbase.HRegionInfo;
- import org.apache.hadoop.hbase.KeyValue;
- import org.apache.hadoop.hbase.TableName;
- import org.apache.hadoop.hbase.Tag;
- import org.apache.hadoop.hbase.TagType;
++import org.apache.hadoop.hbase.*;
 +import org.apache.hadoop.hbase.backup.HFileArchiver;
 +import org.apache.hadoop.hbase.client.Scan;
 +import org.apache.hadoop.hbase.io.HFileLink;
 +import org.apache.hadoop.hbase.io.compress.Compression;
 +import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 +import org.apache.hadoop.hbase.io.hfile.HFile;
 +import org.apache.hadoop.hbase.io.hfile.HFileContext;
 +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
 +import org.apache.hadoop.hbase.master.TableLockManager;
 +import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
 +import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactor;
 +import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactor;
 +import org.apache.hadoop.hbase.regionserver.BloomType;
 +import org.apache.hadoop.hbase.regionserver.HStore;
 +import org.apache.hadoop.hbase.regionserver.StoreFile;
- import org.apache.hadoop.hbase.util.Bytes;
- import org.apache.hadoop.hbase.util.FSUtils;
- import org.apache.hadoop.hbase.util.ReflectionUtils;
- import org.apache.hadoop.hbase.util.Threads;
++import org.apache.hadoop.hbase.util.*;
 +
 +/**
 + * The mob utilities
 + */
 +@InterfaceAudience.Private
 +public class MobUtils {
 +
 +  private static final Log LOG = LogFactory.getLog(MobUtils.class);
 +
 +  private static final ThreadLocal<SimpleDateFormat> LOCAL_FORMAT =
 +      new ThreadLocal<SimpleDateFormat>() {
 +    @Override
 +    protected SimpleDateFormat initialValue() {
 +      return new SimpleDateFormat("yyyyMMdd");
 +    }
 +  };
 +
 +  /**
 +   * Formats a date to a string.
 +   * @param date The date.
 +   * @return The string format of the date, it's yyyymmdd.
 +   */
 +  public static String formatDate(Date date) {
 +    return LOCAL_FORMAT.get().format(date);
 +  }
 +
 +  /**
 +   * Parses the string to a date.
 +   * @param dateString The string format of a date, it's yyyymmdd.
 +   * @return A date.
 +   * @throws ParseException
 +   */
 +  public static Date parseDate(String dateString) throws ParseException {
 +    return LOCAL_FORMAT.get().parse(dateString);
 +  }
 +
 +  /**
 +   * Whether the current cell is a mob reference cell.
 +   * @param cell The current cell.
 +   * @return True if the cell has a mob reference tag, false if it doesn't.
 +   */
 +  public static boolean isMobReferenceCell(Cell cell) {
 +    if (cell.getTagsLength() > 0) {
 +      Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
 +          TagType.MOB_REFERENCE_TAG_TYPE);
 +      return tag != null;
 +    }
 +    return false;
 +  }
 +
 +  /**
 +   * Gets the table name tag.
 +   * @param cell The current cell.
 +   * @return The table name tag.
 +   */
 +  public static Tag getTableNameTag(Cell cell) {
 +    if (cell.getTagsLength() > 0) {
 +      Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
 +          TagType.MOB_TABLE_NAME_TAG_TYPE);
 +      return tag;
 +    }
 +    return null;
 +  }
 +
 +  /**
 +   * Whether the tag list has a mob reference tag.
 +   * @param tags The tag list.
 +   * @return True if the list has a mob reference tag, false if it doesn't.
 +   */
 +  public static boolean hasMobReferenceTag(List<Tag> tags) {
 +    if (!tags.isEmpty()) {
 +      for (Tag tag : tags) {
 +        if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) {
 +          return true;
 +        }
 +      }
 +    }
 +    return false;
 +  }
 +
 +  /**
 +   * Indicates whether it's a raw scan.
 +   * The information is set in the attribute "hbase.mob.scan.raw" of scan.
 +   * For a mob cell, in a normal scan the scanners retrieves the mob cell from the mob file.
 +   * In a raw scan, the scanner directly returns cell in HBase without retrieve the one in
 +   * the mob file.
 +   * @param scan The current scan.
 +   * @return True if it's a raw scan.
 +   */
 +  public static boolean isRawMobScan(Scan scan) {
 +    byte[] raw = scan.getAttribute(MobConstants.MOB_SCAN_RAW);
 +    try {
 +      return raw != null && Bytes.toBoolean(raw);
 +    } catch (IllegalArgumentException e) {
 +      return false;
 +    }
 +  }
 +
 +  /**
 +   * Indicates whether it's a reference only scan.
 +   * The information is set in the attribute "hbase.mob.scan.ref.only" of scan.
 +   * If it's a ref only scan, only the cells with ref tag are returned.
 +   * @param scan The current scan.
 +   * @return True if it's a ref only scan.
 +   */
 +  public static boolean isRefOnlyScan(Scan scan) {
 +    byte[] refOnly = scan.getAttribute(MobConstants.MOB_SCAN_REF_ONLY);
 +    try {
 +      return refOnly != null && Bytes.toBoolean(refOnly);
 +    } catch (IllegalArgumentException e) {
 +      return false;
 +    }
 +  }
 +
 +  /**
 +   * Indicates whether the scan contains the information of caching blocks.
 +   * The information is set in the attribute "hbase.mob.cache.blocks" of scan.
 +   * @param scan The current scan.
 +   * @return True when the Scan attribute specifies to cache the MOB blocks.
 +   */
 +  public static boolean isCacheMobBlocks(Scan scan) {
 +    byte[] cache = scan.getAttribute(MobConstants.MOB_CACHE_BLOCKS);
 +    try {
 +      return cache != null && Bytes.toBoolean(cache);
 +    } catch (IllegalArgumentException e) {
 +      return false;
 +    }
 +  }
 +
 +  /**
 +   * Sets the attribute of caching blocks in the scan.
 +   *
 +   * @param scan
 +   *          The current scan.
 +   * @param cacheBlocks
 +   *          True, set the attribute of caching blocks into the scan, the scanner with this scan
 +   *          caches blocks.
 +   *          False, the scanner doesn't cache blocks for this scan.
 +   */
 +  public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) {
 +    scan.setAttribute(MobConstants.MOB_CACHE_BLOCKS, Bytes.toBytes(cacheBlocks));
 +  }
 +
 +  /**
 +   * Cleans the expired mob files.
 +   * Cleans the files whose creation date is older than (current - columnFamily.ttl), and
 +   * the minVersions of that column family is 0.
 +   * @param fs The current file system.
 +   * @param conf The current configuration.
 +   * @param tableName The current table name.
 +   * @param columnDescriptor The descriptor of the current column family.
 +   * @param cacheConfig The cacheConfig that disables the block cache.
 +   * @param current The current time.
 +   * @throws IOException
 +   */
 +  public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
 +      HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
 +      throws IOException {
 +    long timeToLive = columnDescriptor.getTimeToLive();
 +    if (Integer.MAX_VALUE == timeToLive) {
 +      // no need to clean, because the TTL is not set.
 +      return;
 +    }
 +
 +    Date expireDate = new Date(current - timeToLive * 1000);
 +    expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate());
 +    LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
 +
 +    FileStatus[] stats = null;
 +    Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
 +    Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
 +    try {
 +      stats = fs.listStatus(path);
 +    } catch (FileNotFoundException e) {
 +      LOG.warn("Fail to find the mob file " + path, e);
 +    }
 +    if (null == stats) {
 +      // no file found
 +      return;
 +    }
 +    List<StoreFile> filesToClean = new ArrayList<StoreFile>();
 +    int deletedFileCount = 0;
 +    for (FileStatus file : stats) {
 +      String fileName = file.getPath().getName();
 +      try {
 +        MobFileName mobFileName = null;
 +        if (!HFileLink.isHFileLink(file.getPath())) {
 +          mobFileName = MobFileName.create(fileName);
 +        } else {
 +          HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
 +          mobFileName = MobFileName.create(hfileLink.getOriginPath().getName());
 +        }
 +        Date fileDate = parseDate(mobFileName.getDate());
 +        if (LOG.isDebugEnabled()) {
 +          LOG.debug("Checking file " + fileName);
 +        }
 +        if (fileDate.getTime() < expireDate.getTime()) {
 +          if (LOG.isDebugEnabled()) {
 +            LOG.debug(fileName + " is an expired file");
 +          }
 +          filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
 +        }
 +      } catch (Exception e) {
 +        LOG.error("Cannot parse the fileName " + fileName, e);
 +      }
 +    }
 +    if (!filesToClean.isEmpty()) {
 +      try {
 +        removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(),
 +            filesToClean);
 +        deletedFileCount = filesToClean.size();
 +      } catch (IOException e) {
 +        LOG.error("Fail to delete the mob files " + filesToClean, e);
 +      }
 +    }
 +    LOG.info(deletedFileCount + " expired mob files are deleted");
 +  }
 +
 +  /**
 +   * Gets the root dir of the mob files.
 +   * It's {HBASE_DIR}/mobdir.
 +   * @param conf The current configuration.
 +   * @return the root dir of the mob file.
 +   */
 +  public static Path getMobHome(Configuration conf) {
 +    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
 +    return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
 +  }
 +
 +  /**
 +   * Gets the qualified root dir of the mob files.
 +   * @param conf The current configuration.
 +   * @return The qualified root dir.
 +   * @throws IOException
 +   */
 +  public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
 +    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
 +    Path mobRootDir = new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
 +    FileSystem fs = mobRootDir.getFileSystem(conf);
 +    return mobRootDir.makeQualified(fs);
 +  }
 +
 +  /**
 +   * Gets the region dir of the mob files.
 +   * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
 +   * @param conf The current configuration.
 +   * @param tableName The current table name.
 +   * @return The region dir of the mob files.
 +   */
 +  public static Path getMobRegionPath(Configuration conf, TableName tableName) {
 +    Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
 +    HRegionInfo regionInfo = getMobRegionInfo(tableName);
 +    return new Path(tablePath, regionInfo.getEncodedName());
 +  }
 +
 +  /**
 +   * Gets the family dir of the mob files.
 +   * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
 +   * @param conf The current configuration.
 +   * @param tableName The current table name.
 +   * @param familyName The current family name.
 +   * @return The family dir of the mob files.
 +   */
 +  public static Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
 +    return new Path(getMobRegionPath(conf, tableName), familyName);
 +  }
 +
 +  /**
 +   * Gets the family dir of the mob files.
 +   * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
 +   * @param regionPath The path of mob region which is a dummy one.
 +   * @param familyName The current family name.
 +   * @return The family dir of the mob files.
 +   */
 +  public static Path getMobFamilyPath(Path regionPath, String familyName) {
 +    return new Path(regionPath, familyName);
 +  }
 +
 +  /**
 +   * Gets the HRegionInfo of the mob files.
 +   * This is a dummy region. The mob files are not saved in a region in HBase.
 +   * This is only used in mob snapshot. It's internally used only.
 +   * @param tableName
 +   * @return A dummy mob region info.
 +   */
 +  public static HRegionInfo getMobRegionInfo(TableName tableName) {
 +    HRegionInfo info = new HRegionInfo(tableName, MobConstants.MOB_REGION_NAME_BYTES,
 +        HConstants.EMPTY_END_ROW, false, 0);
 +    return info;
 +  }
 +
 +  /**
 +   * Gets whether the current HRegionInfo is a mob one.
 +   * @param regionInfo The current HRegionInfo.
 +   * @return If true, the current HRegionInfo is a mob one.
 +   */
 +  public static boolean isMobRegionInfo(HRegionInfo regionInfo) {
 +    return regionInfo == null ? false : getMobRegionInfo(regionInfo.getTable()).getEncodedName()
 +        .equals(regionInfo.getEncodedName());
 +  }
 +
 +  /**
 +   * Gets whether the current region name follows the pattern of a mob region name.
 +   * @param tableName The current table name.
 +   * @param regionName The current region name.
 +   * @return True if the current region name follows the pattern of a mob region name.
 +   */
 +  public static boolean isMobRegionName(TableName tableName, byte[] regionName) {
 +    return Bytes.equals(regionName, getMobRegionInfo(tableName).getRegionName());
 +  }
 +
 +  /**
 +   * Gets the working directory of the mob compaction.
 +   * @param root The root directory of the mob compaction.
 +   * @param jobName The current job name.
 +   * @return The directory of the mob compaction for the current job.
 +   */
 +  public static Path getCompactionWorkingPath(Path root, String jobName) {
 +    return new Path(root, jobName);
 +  }
 +
 +  /**
 +   * Archives the mob files.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param tableName The table name.
 +   * @param tableDir The table directory.
 +   * @param family The name of the column family.
 +   * @param storeFiles The files to be deleted.
 +   * @throws IOException
 +   */
 +  public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
 +      Path tableDir, byte[] family, Collection<StoreFile> storeFiles) throws IOException {
 +    HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family,
 +        storeFiles);
 +  }
 +
 +  /**
 +   * Creates a mob reference KeyValue.
 +   * The value of the mob reference KeyValue is mobCellValueSize + mobFileName.
 +   * @param cell The original Cell.
 +   * @param fileName The mob file name where the mob reference KeyValue is written.
 +   * @param tableNameTag The tag of the current table name. It's very important in
 +   *                        cloning the snapshot.
 +   * @return The mob reference KeyValue.
 +   */
 +  public static KeyValue createMobRefKeyValue(Cell cell, byte[] fileName, Tag tableNameTag) {
 +    // Append the tags to the KeyValue.
 +    // The key is same, the value is the filename of the mob file
 +    List<Tag> tags = new ArrayList<Tag>();
 +    // Add the ref tag as the 1st one.
 +    tags.add(MobConstants.MOB_REF_TAG);
 +    // Add the tag of the source table name, this table is where this mob file is flushed
 +    // from.
 +    // It's very useful in cloning the snapshot. When reading from the cloning table, we need to
 +    // find the original mob files by this table name. For details please see cloning
 +    // snapshot for mob files.
 +    tags.add(tableNameTag);
 +    // Add the existing tags.
 +    tags.addAll(Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()));
 +    int valueLength = cell.getValueLength();
 +    byte[] refValue = Bytes.add(Bytes.toBytes(valueLength), fileName);
 +    KeyValue reference = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
 +        cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
 +        cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
 +        cell.getTimestamp(), KeyValue.Type.Put, refValue, 0, refValue.length, tags);
 +    reference.setSequenceId(cell.getSequenceId());
 +    return reference;
 +  }
 +
 +  /**
 +   * Creates a writer for the mob file in temp directory.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param family The descriptor of the current column family.
 +   * @param date The date string, its format is yyyymmmdd.
 +   * @param basePath The basic path for a temp directory.
 +   * @param maxKeyCount The key count.
 +   * @param compression The compression algorithm.
 +   * @param startKey The hex string of the start key.
 +   * @param cacheConfig The current cache config.
 +   * @return The writer for the mob file.
 +   * @throws IOException
 +   */
 +  public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
 +      HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
 +      Compression.Algorithm compression, String startKey, CacheConfig cacheConfig)
 +      throws IOException {
 +    MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
 +        .replaceAll("-", ""));
 +    return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
 +      cacheConfig);
 +  }
 +
 +  /**
 +   * Creates a writer for the ref file in temp directory.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param family The descriptor of the current column family.
 +   * @param basePath The basic path for a temp directory.
 +   * @param maxKeyCount The key count.
 +   * @param cacheConfig The current cache config.
 +   * @return The writer for the mob file.
 +   * @throws IOException
 +   */
 +  public static StoreFile.Writer createRefFileWriter(Configuration conf, FileSystem fs,
 +    HColumnDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig)
 +    throws IOException {
 +    HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(true)
 +      .withIncludesTags(true).withCompression(family.getCompactionCompression())
-       .withCompressTags(family.shouldCompressTags()).withChecksumType(HStore.getChecksumType(conf))
++      .withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf))
 +      .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
 +      .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build();
 +    Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", ""));
 +    StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath)
-       .withComparator(KeyValue.COMPARATOR).withBloomType(family.getBloomFilterType())
++      .withComparator(CellComparator.COMPARATOR).withBloomType(family.getBloomFilterType())
 +      .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
 +    return w;
 +  }
 +
 +  /**
 +   * Creates a writer for the mob file in temp directory.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param family The descriptor of the current column family.
 +   * @param date The date string, its format is yyyymmmdd.
 +   * @param basePath The basic path for a temp directory.
 +   * @param maxKeyCount The key count.
 +   * @param compression The compression algorithm.
 +   * @param startKey The start key.
 +   * @param cacheConfig The current cache config.
 +   * @return The writer for the mob file.
 +   * @throws IOException
 +   */
 +  public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
 +      HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
 +      Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig)
 +      throws IOException {
 +    MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
 +        .replaceAll("-", ""));
 +    return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
 +      cacheConfig);
 +  }
 +
 +  /**
 +   * Creates a writer for the del file in temp directory.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param family The descriptor of the current column family.
 +   * @param date The date string, its format is yyyymmmdd.
 +   * @param basePath The basic path for a temp directory.
 +   * @param maxKeyCount The key count.
 +   * @param compression The compression algorithm.
 +   * @param startKey The start key.
 +   * @param cacheConfig The current cache config.
 +   * @return The writer for the del file.
 +   * @throws IOException
 +   */
 +  public static StoreFile.Writer createDelFileWriter(Configuration conf, FileSystem fs,
 +      HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
 +      Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig)
 +      throws IOException {
 +    String suffix = UUID
 +      .randomUUID().toString().replaceAll("-", "") + "_del";
 +    MobFileName mobFileName = MobFileName.create(startKey, date, suffix);
 +    return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
 +      cacheConfig);
 +  }
 +
 +  /**
 +   * Creates a writer for the del file in temp directory.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param family The descriptor of the current column family.
 +   * @param mobFileName The mob file name.
 +   * @param basePath The basic path for a temp directory.
 +   * @param maxKeyCount The key count.
 +   * @param compression The compression algorithm.
 +   * @param cacheConfig The current cache config.
 +   * @return The writer for the mob file.
 +   * @throws IOException
 +   */
 +  private static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
 +    HColumnDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
 +    Compression.Algorithm compression, CacheConfig cacheConfig) throws IOException {
 +    HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
-       .withIncludesMvcc(true).withIncludesTags(true).withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
++      .withIncludesMvcc(true).withIncludesTags(true)
++      .withChecksumType(ChecksumType.getDefaultChecksumType())
 +      .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).withBlockSize(family.getBlocksize())
 +      .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build();
 +
 +    StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs)
 +      .withFilePath(new Path(basePath, mobFileName.getFileName()))
-       .withComparator(KeyValue.COMPARATOR).withBloomType(BloomType.NONE)
++      .withComparator(CellComparator.COMPARATOR).withBloomType(BloomType.NONE)
 +      .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
 +    return w;
 +  }
 +
 +  /**
 +   * Commits the mob file.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param sourceFile The path where the mob file is saved.
 +   * @param targetPath The directory path where the source file is renamed to.
 +   * @param cacheConfig The current cache config.
 +   * @return The target file path the source file is renamed to.
 +   * @throws IOException
 +   */
 +  public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
 +      Path targetPath, CacheConfig cacheConfig) throws IOException {
 +    if (sourceFile == null) {
 +      return null;
 +    }
 +    Path dstPath = new Path(targetPath, sourceFile.getName());
 +    validateMobFile(conf, fs, sourceFile, cacheConfig);
 +    String msg = "Renaming flushed file from " + sourceFile + " to " + dstPath;
 +    LOG.info(msg);
 +    Path parent = dstPath.getParent();
 +    if (!fs.exists(parent)) {
 +      fs.mkdirs(parent);
 +    }
 +    if (!fs.rename(sourceFile, dstPath)) {
 +      throw new IOException("Failed rename of " + sourceFile + " to " + dstPath);
 +    }
 +    return dstPath;
 +  }
 +
 +  /**
 +   * Validates a mob file by opening and closing it.
 +   * @param conf The current configuration.
 +   * @param fs The current file system.
 +   * @param path The path where the mob file is saved.
 +   * @param cacheConfig The current cache config.
 +   */
 +  private static void validateMobFile(Configuration conf, FileSystem fs, Path path,
 +      CacheConfig cacheConfig) throws IOException {
 +    StoreFile storeFile = null;
 +    try {
 +      storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE);
 +      storeFile.createReader();
 +    } catch (IOException e) {
 +      LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e);
 +      throw e;
 +    } finally {
 +      if (storeFile != null) {
 +        storeFile.closeReader(false);
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Indicates whether the current mob ref cell has a valid value.
 +   * A mob ref cell has a mob reference tag.
 +   * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
 +   * The real mob value length takes 4 bytes.
 +   * The remaining part is the mob file name.
 +   * @param cell The mob ref cell.
 +   * @return True if the cell has a valid value.
 +   */
 +  public static boolean hasValidMobRefCellValue(Cell cell) {
 +    return cell.getValueLength() > Bytes.SIZEOF_INT;
 +  }
 +
 +  /**
 +   * Gets the mob value length from the mob ref cell.
 +   * A mob ref cell has a mob reference tag.
 +   * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
 +   * The real mob value length takes 4 bytes.
 +   * The remaining part is the mob file name.
 +   * @param cell The mob ref cell.
 +   * @return The real mob value length.
 +   */
 +  public static int getMobValueLength(Cell cell) {
 +    return Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), Bytes.SIZEOF_INT);
 +  }
 +
 +  /**
 +   * Gets the mob file name from the mob ref cell.
 +   * A mob ref cell has a mob reference tag.
 +   * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
 +   * The real mob value length takes 4 bytes.
 +   * The remaining part is the mob file name.
 +   * @param cell The mob ref cell.
 +   * @return The mob file name.
 +   */
 +  public static String getMobFileName(Cell cell) {
 +    return Bytes.toString(cell.getValueArray(), cell.getValueOffset() + Bytes.SIZEOF_INT,
 +        cell.getValueLength() - Bytes.SIZEOF_INT);
 +  }
 +
 +  /**
 +   * Gets the table name used in the table lock.
 +   * The table lock name is a dummy one, it's not a table name. It's tableName + ".mobLock".
 +   * @param tn The table name.
 +   * @return The table name used in table lock.
 +   */
 +  public static TableName getTableLockName(TableName tn) {
 +    byte[] tableName = tn.getName();
 +    return TableName.valueOf(Bytes.add(tableName, MobConstants.MOB_TABLE_LOCK_SUFFIX));
 +  }
 +
 +  /**
 +   * Performs the mob file compaction.
 +   * @param conf the Configuration
 +   * @param fs the file system
 +   * @param tableName the table the compact
 +   * @param hcd the column descriptor
 +   * @param pool the thread pool
 +   * @param tableLockManager the tableLock manager
 +   * @param isForceAllFiles Whether add all mob files into the compaction.
 +   */
 +  public static void doMobFileCompaction(Configuration conf, FileSystem fs, TableName tableName,
 +    HColumnDescriptor hcd, ExecutorService pool, TableLockManager tableLockManager,
 +    boolean isForceAllFiles) throws IOException {
 +    String className = conf.get(MobConstants.MOB_FILE_COMPACTOR_CLASS_KEY,
 +      PartitionedMobFileCompactor.class.getName());
 +    // instantiate the mob file compactor.
 +    MobFileCompactor compactor = null;
 +    try {
 +      compactor = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
 +        Configuration.class, FileSystem.class, TableName.class, HColumnDescriptor.class,
 +        ExecutorService.class }, new Object[] { conf, fs, tableName, hcd, pool });
 +    } catch (Exception e) {
 +      throw new IOException("Unable to load configured mob file compactor '" + className + "'", e);
 +    }
 +    // compact only for mob-enabled column.
 +    // obtain a write table lock before performing compaction to avoid race condition
 +    // with major compaction in mob-enabled column.
 +    boolean tableLocked = false;
 +    TableLock lock = null;
 +    try {
 +      // the tableLockManager might be null in testing. In that case, it is lock-free.
 +      if (tableLockManager != null) {
 +        lock = tableLockManager.writeLock(MobUtils.getTableLockName(tableName),
 +          "Run MobFileCompaction");
 +        lock.acquire();
 +      }
 +      tableLocked = true;
 +      compactor.compact(isForceAllFiles);
 +    } catch (Exception e) {
 +      LOG.error("Fail to compact the mob files for the column " + hcd.getNameAsString()
 +        + " in the table " + tableName.getNameAsString(), e);
 +    } finally {
 +      if (lock != null && tableLocked) {
 +        try {
 +          lock.release();
 +        } catch (IOException e) {
 +          LOG.error("Fail to release the write lock for the table " + tableName.getNameAsString(),
 +            e);
 +        }
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Creates a thread pool.
 +   * @param conf the Configuration
 +   * @return A thread pool.
 +   */
 +  public static ExecutorService createMobFileCompactorThreadPool(Configuration conf) {
 +    int maxThreads = conf.getInt(MobConstants.MOB_FILE_COMPACTION_THREADS_MAX,
 +      MobConstants.DEFAULT_MOB_FILE_COMPACTION_THREADS_MAX);
 +    if (maxThreads == 0) {
 +      maxThreads = 1;
 +    }
 +    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
 +    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue,
 +      Threads.newDaemonThreadFactory("MobFileCompactor"), new RejectedExecutionHandler() {
 +        @Override
 +        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
 +          try {
 +            // waiting for a thread to pick up instead of throwing exceptions.
 +            queue.put(r);
 +          } catch (InterruptedException e) {
 +            throw new RejectedExecutionException(e);
 +          }
 +        }
 +      });
 +    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
 +    return pool;
 +  }
 +}


Mime
View raw message