Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C609B200D29 for ; Wed, 11 Oct 2017 17:13:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id C428D160BE5; Wed, 11 Oct 2017 15:13:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4BD72160BEC for ; Wed, 11 Oct 2017 17:13:10 +0200 (CEST) Received: (qmail 8560 invoked by uid 500); 11 Oct 2017 15:13:08 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 7378 invoked by uid 99); 11 Oct 2017 15:13:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Oct 2017 15:13:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 85805DFBC7; Wed, 11 Oct 2017 15:13:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Wed, 11 Oct 2017 15:13:15 -0000 Message-Id: <26c129f5a431415e881dcb5d9ef2eaaf@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [10/51] [partial] hbase-site git commit: Published site at . archived-at: Wed, 11 Oct 2017 15:13:13 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c0571676/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html index 41e0c24..eac35d3 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html @@ -110,8004 +110,8178 @@ 102import org.apache.hadoop.hbase.UnknownScannerException; 103import org.apache.hadoop.hbase.client.Append; 104import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -105import org.apache.hadoop.hbase.client.Delete; -106import org.apache.hadoop.hbase.client.Durability; -107import org.apache.hadoop.hbase.client.Get; -108import org.apache.hadoop.hbase.client.Increment; -109import org.apache.hadoop.hbase.client.IsolationLevel; -110import org.apache.hadoop.hbase.client.Mutation; -111import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; -112import org.apache.hadoop.hbase.client.Put; -113import org.apache.hadoop.hbase.client.RegionInfo; -114import org.apache.hadoop.hbase.client.RegionReplicaUtil; -115import org.apache.hadoop.hbase.client.Result; -116import org.apache.hadoop.hbase.client.RowMutations; -117import org.apache.hadoop.hbase.client.Scan; -118import org.apache.hadoop.hbase.client.TableDescriptor; -119import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -120import org.apache.hadoop.hbase.conf.ConfigurationManager; -121import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; -122import org.apache.hadoop.hbase.coprocessor.RegionObserver.MutationType; -123import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; -124import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; -125import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; -126import org.apache.hadoop.hbase.exceptions.TimeoutIOException; -127import org.apache.hadoop.hbase.exceptions.UnknownProtocolException; -128import org.apache.hadoop.hbase.filter.ByteArrayComparable; -129import org.apache.hadoop.hbase.filter.FilterWrapper; -130import org.apache.hadoop.hbase.filter.IncompatibleFilterException; -131import org.apache.hadoop.hbase.io.HFileLink; -132import org.apache.hadoop.hbase.io.HeapSize; -133import org.apache.hadoop.hbase.io.TimeRange; -134import org.apache.hadoop.hbase.io.hfile.HFile; -135import org.apache.hadoop.hbase.ipc.CallerDisconnectedException; -136import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -137import org.apache.hadoop.hbase.ipc.RpcCall; -138import org.apache.hadoop.hbase.ipc.RpcServer; -139import org.apache.hadoop.hbase.monitoring.MonitoredTask; -140import org.apache.hadoop.hbase.monitoring.TaskMonitor; -141import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry; -142import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; -143import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState; -144import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; -145import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; -146import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; -147import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; -148import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; -149import org.apache.hadoop.hbase.regionserver.wal.WALUtil; -150import org.apache.hadoop.hbase.security.User; -151import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -152import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -153import org.apache.hadoop.hbase.util.Bytes; -154import org.apache.hadoop.hbase.util.CancelableProgressable; -155import org.apache.hadoop.hbase.util.ClassSize; -156import org.apache.hadoop.hbase.util.CollectionUtils; -157import org.apache.hadoop.hbase.util.CompressionTest; -158import org.apache.hadoop.hbase.util.EncryptionTest; -159import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -160import org.apache.hadoop.hbase.util.FSUtils; -161import org.apache.hadoop.hbase.util.HashedBytes; -162import org.apache.hadoop.hbase.util.Pair; -163import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; -164import org.apache.hadoop.hbase.util.Threads; -165import org.apache.hadoop.hbase.wal.WAL; -166import org.apache.hadoop.hbase.wal.WALEdit; -167import org.apache.hadoop.hbase.wal.WALFactory; -168import org.apache.hadoop.hbase.wal.WALKey; -169import org.apache.hadoop.hbase.wal.WALSplitter; -170import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay; -171import org.apache.hadoop.io.MultipleIOException; -172import org.apache.hadoop.util.StringUtils; -173import org.apache.htrace.Trace; -174import org.apache.htrace.TraceScope; -175import org.apache.yetus.audience.InterfaceAudience; -176 -177import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; -178import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions; -179import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -180import org.apache.hadoop.hbase.shaded.com.google.common.collect.Maps; -181import org.apache.hadoop.hbase.shaded.com.google.common.io.Closeables; -182import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; -183import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; -184import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -185import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; -186import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; -187import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall; -188import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad; -189import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId; -190import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -191import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; -192import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; -193import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; -194import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; -195import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; -196import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; -197import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType; -198import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; -199 -200@SuppressWarnings("deprecation") -201@InterfaceAudience.Private -202public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region { -203 private static final Log LOG = LogFactory.getLog(HRegion.class); -204 -205 public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = -206 "hbase.hregion.scan.loadColumnFamiliesOnDemand"; -207 -208 public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; -209 public static final int DEFAULT_MAX_CELL_SIZE = 10485760; -210 -211 /** -212 * This is the global default value for durability. All tables/mutations not -213 * defining a durability or using USE_DEFAULT will default to this value. -214 */ -215 private static final Durability DEFAULT_DURABILITY = Durability.SYNC_WAL; -216 -217 final AtomicBoolean closed = new AtomicBoolean(false); -218 -219 /* Closing can take some time; use the closing flag if there is stuff we don't -220 * want to do while in closing state; e.g. like offer this region up to the -221 * master as a region to close if the carrying regionserver is overloaded. -222 * Once set, it is never cleared. -223 */ -224 final AtomicBoolean closing = new AtomicBoolean(false); -225 -226 /** -227 * The max sequence id of flushed data on this region. There is no edit in memory that is -228 * less that this sequence id. -229 */ -230 private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM; -231 -232 /** -233 * Record the sequence id of last flush operation. Can be in advance of -234 * {@link #maxFlushedSeqId} when flushing a single column family. In this case, -235 * {@link #maxFlushedSeqId} will be older than the oldest edit in memory. -236 */ -237 private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM; -238 -239 /** -240 * The sequence id of the last replayed open region event from the primary region. This is used -241 * to skip entries before this due to the possibility of replay edits coming out of order from -242 * replication. -243 */ -244 protected volatile long lastReplayedOpenRegionSeqId = -1L; -245 protected volatile long lastReplayedCompactionSeqId = -1L; -246 -247 ////////////////////////////////////////////////////////////////////////////// -248 // Members -249 ////////////////////////////////////////////////////////////////////////////// -250 -251 // map from a locked row to the context for that lock including: -252 // - CountDownLatch for threads waiting on that row -253 // - the thread that owns the lock (allow reentrancy) -254 // - reference count of (reentrant) locks held by the thread -255 // - the row itself -256 private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows = -257 new ConcurrentHashMap<>(); -258 -259 protected final Map<byte[], HStore> stores = -260 new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR); -261 -262 // TODO: account for each registered handler in HeapSize computation -263 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); -264 -265 private final AtomicLong memstoreDataSize = new AtomicLong(0);// Track data size in all memstores -266 private final RegionServicesForStores regionServicesForStores = new RegionServicesForStores(this); -267 -268 // Debug possible data loss due to WAL off -269 final LongAdder numMutationsWithoutWAL = new LongAdder(); -270 final LongAdder dataInMemoryWithoutWAL = new LongAdder(); -271 -272 // Debug why CAS operations are taking a while. -273 final LongAdder checkAndMutateChecksPassed = new LongAdder(); -274 final LongAdder checkAndMutateChecksFailed = new LongAdder(); -275 -276 // Number of requests -277 final LongAdder readRequestsCount = new LongAdder(); -278 final LongAdder filteredReadRequestsCount = new LongAdder(); -279 final LongAdder writeRequestsCount = new LongAdder(); +105import org.apache.hadoop.hbase.client.CompactionState; +106import org.apache.hadoop.hbase.client.Delete; +107import org.apache.hadoop.hbase.client.Durability; +108import org.apache.hadoop.hbase.client.Get; +109import org.apache.hadoop.hbase.client.Increment; +110import org.apache.hadoop.hbase.client.IsolationLevel; +111import org.apache.hadoop.hbase.client.Mutation; +112import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; +113import org.apache.hadoop.hbase.client.Put; +114import org.apache.hadoop.hbase.client.RegionInfo; +115import org.apache.hadoop.hbase.client.RegionReplicaUtil; +116import org.apache.hadoop.hbase.client.Result; +117import org.apache.hadoop.hbase.client.RowMutations; +118import org.apache.hadoop.hbase.client.Scan; +119import org.apache.hadoop.hbase.client.TableDescriptor; +120import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +121import org.apache.hadoop.hbase.conf.ConfigurationManager; +122import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; +123import org.apache.hadoop.hbase.coprocessor.RegionObserver.MutationType; +124import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; +125import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; +126import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; +127import org.apache.hadoop.hbase.exceptions.TimeoutIOException; +128import org.apache.hadoop.hbase.exceptions.UnknownProtocolException; +129import org.apache.hadoop.hbase.filter.ByteArrayComparable; +130import org.apache.hadoop.hbase.filter.FilterWrapper; +131import org.apache.hadoop.hbase.filter.IncompatibleFilterException; +132import org.apache.hadoop.hbase.io.HFileLink; +133import org.apache.hadoop.hbase.io.HeapSize; +134import org.apache.hadoop.hbase.io.TimeRange; +135import org.apache.hadoop.hbase.io.hfile.HFile; +136import org.apache.hadoop.hbase.ipc.CallerDisconnectedException; +137import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +138import org.apache.hadoop.hbase.ipc.RpcCall; +139import org.apache.hadoop.hbase.ipc.RpcServer; +140import org.apache.hadoop.hbase.monitoring.MonitoredTask; +141import org.apache.hadoop.hbase.monitoring.TaskMonitor; +142import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry; +143import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; +144import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState; +145import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +146import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; +147import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; +148import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; +149import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; +150import org.apache.hadoop.hbase.regionserver.wal.WALUtil; +151import org.apache.hadoop.hbase.security.User; +152import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +153import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +154import org.apache.hadoop.hbase.util.Bytes; +155import org.apache.hadoop.hbase.util.CancelableProgressable; +156import org.apache.hadoop.hbase.util.ClassSize; +157import org.apache.hadoop.hbase.util.CollectionUtils; +158import org.apache.hadoop.hbase.util.CompressionTest; +159import org.apache.hadoop.hbase.util.EncryptionTest; +160import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +161import org.apache.hadoop.hbase.util.FSUtils; +162import org.apache.hadoop.hbase.util.HashedBytes; +163import org.apache.hadoop.hbase.util.Pair; +164import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +165import org.apache.hadoop.hbase.util.Threads; +166import org.apache.hadoop.hbase.wal.WAL; +167import org.apache.hadoop.hbase.wal.WALEdit; +168import org.apache.hadoop.hbase.wal.WALFactory; +169import org.apache.hadoop.hbase.wal.WALKey; +170import org.apache.hadoop.hbase.wal.WALSplitter; +171import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay; +172import org.apache.hadoop.io.MultipleIOException; +173import org.apache.hadoop.util.StringUtils; +174import org.apache.htrace.Trace; +175import org.apache.htrace.TraceScope; +176import org.apache.yetus.audience.InterfaceAudience; +177 +178import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; +179import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions; +180import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; +181import org.apache.hadoop.hbase.shaded.com.google.common.collect.Maps; +182import org.apache.hadoop.hbase.shaded.com.google.common.io.Closeables; +183import org.apache.hadoop.hbase.shaded.com.google.protobuf.Service; +184import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; +185import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; +186import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +187import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; +188import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall; +189import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad; +190import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId; +191import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; +192import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; +193import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; +194import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; +195import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; +196import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; +197import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; +198import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType; +199import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; +200 +201/** +202 * Regions store data for a certain region of a table. It stores all columns +203 * for each row. A given table consists of one or more Regions. +204 * +205 * <p>An Region is defined by its table and its key extent. +206 * +207 * <p>Locking at the Region level serves only one purpose: preventing the +208 * region from being closed (and consequently split) while other operations +209 * are ongoing. Each row level operation obtains both a row lock and a region +210 * read lock for the duration of the operation. While a scanner is being +211 * constructed, getScanner holds a read lock. If the scanner is successfully +212 * constructed, it holds a read lock until it is closed. A close takes out a +213 * write lock and consequently will block for ongoing operations and will block +214 * new operations from starting while the close is in progress. +215 */ +216@SuppressWarnings("deprecation") +217@InterfaceAudience.Private +218public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region { +219 private static final Log LOG = LogFactory.getLog(HRegion.class); +220 +221 public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = +222 "hbase.hregion.scan.loadColumnFamiliesOnDemand"; +223 +224 public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; +225 public static final int DEFAULT_MAX_CELL_SIZE = 10485760; +226 +227 /** +228 * This is the global default value for durability. All tables/mutations not +229 * defining a durability or using USE_DEFAULT will default to this value. +230 */ +231 private static final Durability DEFAULT_DURABILITY = Durability.SYNC_WAL; +232 +233 final AtomicBoolean closed = new AtomicBoolean(false); +234 +235 /* Closing can take some time; use the closing flag if there is stuff we don't +236 * want to do while in closing state; e.g. like offer this region up to the +237 * master as a region to close if the carrying regionserver is overloaded. +238 * Once set, it is never cleared. +239 */ +240 final AtomicBoolean closing = new AtomicBoolean(false); +241 +242 /** +243 * The max sequence id of flushed data on this region. There is no edit in memory that is +244 * less that this sequence id. +245 */ +246 private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM; +247 +248 /** +249 * Record the sequence id of last flush operation. Can be in advance of +250 * {@link #maxFlushedSeqId} when flushing a single column family. In this case, +251 * {@link #maxFlushedSeqId} will be older than the oldest edit in memory. +252 */ +253 private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM; +254 +255 /** +256 * The sequence id of the last replayed open region event from the primary region. This is used +257 * to skip entries before this due to the possibility of replay edits coming out of order from +258 * replication. +259 */ +260 protected volatile long lastReplayedOpenRegionSeqId = -1L; +261 protected volatile long lastReplayedCompactionSeqId = -1L; +262 +263 ////////////////////////////////////////////////////////////////////////////// +264 // Members +265 ////////////////////////////////////////////////////////////////////////////// +266 +267 // map from a locked row to the context for that lock including: +268 // - CountDownLatch for threads waiting on that row +269 // - the thread that owns the lock (allow reentrancy) +270 // - reference count of (reentrant) locks held by the thread +271 // - the row itself +272 private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows = +273 new ConcurrentHashMap<>(); +274 +275 protected final Map<byte[], HStore> stores = +276 new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR); +277 +278 // TODO: account for each registered handler in HeapSize computation +279 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); 280 -281 // Number of requests blocked by memstore size. -282 private final LongAdder blockedRequestsCount = new LongAdder(); +281 private final AtomicLong memstoreDataSize = new AtomicLong(0);// Track data size in all memstores +282 private final RegionServicesForStores regionServicesForStores = new RegionServicesForStores(this); 283 -284 // Compaction LongAdders -285 final LongAdder compactionsFinished = new LongAdder(); -286 final LongAdder compactionsFailed = new LongAdder(); -287 final LongAdder compactionNumFilesCompacted = new LongAdder(); -288 final LongAdder compactionNumBytesCompacted = new LongAdder(); -289 final LongAdder compactionsQueued = new LongAdder(); -290 final LongAdder flushesQueued = new LongAdder(); +284 // Debug possible data loss due to WAL off +285 final LongAdder numMutationsWithoutWAL = new LongAdder(); +286 final LongAdder dataInMemoryWithoutWAL = new LongAdder(); +287 +288 // Debug why CAS operations are taking a while. +289 final LongAdder checkAndMutateChecksPassed = new LongAdder(); +290 final LongAdder checkAndMutateChecksFailed = new LongAdder(); 291 -292 private final WAL wal; -293 private final HRegionFileSystem fs; -294 protected final Configuration conf; -295 private final Configuration baseConf; -296 private final int rowLockWaitDuration; -297 static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000; -298 -299 // The internal wait duration to acquire a lock before read/update -300 // from the region. It is not per row. The purpose of this wait time -301 // is to avoid waiting a long time while the region is busy, so that -302 // we can release the IPC handler soon enough to improve the -303 // availability of the region server. It can be adjusted by -304 // tuning configuration "hbase.busy.wait.duration". -305 final long busyWaitDuration; -306 static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT; +292 // Number of requests +293 final LongAdder readRequestsCount = new LongAdder(); +294 final LongAdder filteredReadRequestsCount = new LongAdder(); +295 final LongAdder writeRequestsCount = new LongAdder(); +296 +297 // Number of requests blocked by memstore size. +298 private final LongAdder blockedRequestsCount = new LongAdder(); +299 +300 // Compaction LongAdders +301 final LongAdder compactionsFinished = new LongAdder(); +302 final LongAdder compactionsFailed = new LongAdder(); +303 final LongAdder compactionNumFilesCompacted = new LongAdder(); +304 final LongAdder compactionNumBytesCompacted = new LongAdder(); +305 final LongAdder compactionsQueued = new LongAdder(); +306 final LongAdder flushesQueued = new LongAdder(); 307 -308 // If updating multiple rows in one call, wait longer, -309 // i.e. waiting for busyWaitDuration * # of rows. However, -310 // we can limit the max multiplier. -311 final int maxBusyWaitMultiplier; -312 -313 // Max busy wait duration. There is no point to wait longer than the RPC -314 // purge timeout, when a RPC call will be terminated by the RPC engine. -315 final long maxBusyWaitDuration; -316 -317 // Max cell size. If nonzero, the maximum allowed size for any given cell -318 // in bytes -319 final long maxCellSize; -320 -321 // negative number indicates infinite timeout -322 static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L; -323 final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool(); -324 -325 private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints; -326 -327 /** -328 * The sequence ID that was enLongAddered when this region was opened. -329 */ -330 private long openSeqNum = HConstants.NO_SEQNUM; -331 -332 /** -333 * The default setting for whether to enable on-demand CF loading for -334 * scan requests to this region. Requests can override it. -335 */ -336 private boolean isLoadingCfsOnDemandDefault = false; -337 -338 private final AtomicInteger majorInProgress = new AtomicInteger(0); -339 private final AtomicInteger minorInProgress = new AtomicInteger(0); +308 private final WAL wal; +309 private final HRegionFileSystem fs; +310 protected final Configuration conf; +311 private final Configuration baseConf; +312 private final int rowLockWaitDuration; +313 static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000; +314 +315 // The internal wait duration to acquire a lock before read/update +316 // from the region. It is not per row. The purpose of this wait time +317 // is to avoid waiting a long time while the region is busy, so that +318 // we can release the IPC handler soon enough to improve the +319 // availability of the region server. It can be adjusted by +320 // tuning configuration "hbase.busy.wait.duration". +321 final long busyWaitDuration; +322 static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT; +323 +324 // If updating multiple rows in one call, wait longer, +325 // i.e. waiting for busyWaitDuration * # of rows. However, +326 // we can limit the max multiplier. +327 final int maxBusyWaitMultiplier; +328 +329 // Max busy wait duration. There is no point to wait longer than the RPC +330 // purge timeout, when a RPC call will be terminated by the RPC engine. +331 final long maxBusyWaitDuration; +332 +333 // Max cell size. If nonzero, the maximum allowed size for any given cell +334 // in bytes +335 final long maxCellSize; +336 +337 // negative number indicates infinite timeout +338 static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L; +339 final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool(); 340 -341 // -342 // Context: During replay we want to ensure that we do not lose any data. So, we -343 // have to be conservative in how we replay wals. For each store, we calculate -344 // the maxSeqId up to which the store was flushed. And, skip the edits which -345 // are equal to or lower than maxSeqId for each store. -346 // The following map is populated when opening the region -347 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); -348 -349 /** Saved state from replaying prepare flush cache */ -350 private PrepareFlushResult prepareFlushResult = null; -351 -352 /** -353 * Config setting for whether to allow writes when a region is in recovering or not. -354 */ -355 private boolean disallowWritesInRecovering = false; +341 private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints; +342 +343 /** +344 * The sequence ID that was enLongAddered when this region was opened. +345 */ +346 private long openSeqNum = HConstants.NO_SEQNUM; +347 +348 /** +349 * The default setting for whether to enable on-demand CF loading for +350 * scan requests to this region. Requests can override it. +351 */ +352 private boolean isLoadingCfsOnDemandDefault = false; +353 +354 private final AtomicInteger majorInProgress = new AtomicInteger(0); +355 private final AtomicInteger minorInProgress = new AtomicInteger(0); 356 -357 // When a region is in recovering state, it can only accept writes not reads -358 private volatile boolean recovering = false; -359 -360 private volatile Optional<ConfigurationManager> configurationManager; -361 -362 // Used for testing. -363 private volatile Long timeoutForWriteLock = null; +357 // +358 // Context: During replay we want to ensure that we do not lose any data. So, we +359 // have to be conservative in how we replay wals. For each store, we calculate +360 // the maxSeqId up to which the store was flushed. And, skip the edits which +361 // are equal to or lower than maxSeqId for each store. +362 // The following map is populated when opening the region +363 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 364 -365 /** -366 * @return The smallest mvcc readPoint across all the scanners in this -367 * region. Writes older than this readPoint, are included in every -368 * read operation. -369 */ -370 public long getSmallestReadPoint() { -371 long minimumReadPoint; -372 // We need to ensure that while we are calculating the smallestReadPoint -373 // no new RegionScanners can grab a readPoint that we are unaware of. -374 // We achieve this by synchronizing on the scannerReadPoints object. -375 synchronized (scannerReadPoints) { -376 minimumReadPoint = mvcc.getReadPoint(); -377 for (Long readPoint : this.scannerReadPoints.values()) { -378 if (readPoint < minimumReadPoint) { -379 minimumReadPoint = readPoint; -380 } -381 } -382 } -383 return minimumReadPoint; -384 } -385 -386 /* -387 * Data structure of write state flags used coordinating flushes, -388 * compactions and closes. -389 */ -390 static class WriteState { -391 // Set while a memstore flush is happening. -392 volatile boolean flushing = false; -393 // Set when a flush has been requested. -394 volatile boolean flushRequested = false; -395 // Number of compactions running. -396 AtomicInteger compacting = new AtomicInteger(0); -397 // Gets set in close. If set, cannot compact or flush again. -398 volatile boolean writesEnabled = true; -399 // Set if region is read-only -400 volatile boolean readOnly = false; -401 // whether the reads are enabled. This is different than readOnly, because readOnly is -402 // static in the lifetime of the region, while readsEnabled is dynamic -403 volatile boolean readsEnabled = true; -404 -405 /** -406 * Set flags that make this region read-only. -407 * -408 * @param onOff flip value for region r/o setting -409 */ -410 synchronized void setReadOnly(final boolean onOff) { -411 this.writesEnabled = !onOff; -412 this.readOnly = onOff; -413 } -414 -415 boolean isReadOnly() { -416 return this.readOnly; -417 } -418 -419 boolean isFlushRequested() { -420 return this.flushRequested; -421 } -422 -423 void setReadsEnabled(boolean readsEnabled) { -424 this.readsEnabled = readsEnabled; -425 } -426 -427 static final long HEAP_SIZE = ClassSize.align( -428 ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN); -429 } +365 /** Saved state from replaying prepare flush cache */ +366 private PrepareFlushResult prepareFlushResult = null; +367 +368 /** +369 * Config setting for whether to allow writes when a region is in recovering or not. +370 */ +371 private boolean disallowWritesInRecovering = false; +372 +373 // When a region is in recovering state, it can only accept writes not reads +374 private volatile boolean recovering = false; +375 +376 private volatile Optional<ConfigurationManager> configurationManager; +377 +378 // Used for testing. +379 private volatile Long timeoutForWriteLock = null; +380 +381 /** +382 * @return The smallest mvcc readPoint across all the scanners in this +383 * region. Writes older than this readPoint, are included in every +384 * read operation. +385 */ +386 public long getSmallestReadPoint() { +387 long minimumReadPoint; +388 // We need to ensure that while we are calculating the smallestReadPoint +389 // no new RegionScanners can grab a readPoint that we are unaware of. +390 // We achieve this by synchronizing on the scannerReadPoints object. +391 synchronized (scannerReadPoints) { +392 minimumReadPoint = mvcc.getReadPoint(); +393 for (Long readPoint : this.scannerReadPoints.values()) { +394 if (readPoint < minimumReadPoint) { +395 minimumReadPoint = readPoint; +396 } +397 } +398 } +399 return minimumReadPoint; +400 } +401 +402 /* +403 * Data structure of write state flags used coordinating flushes, +404 * compactions and closes. +405 */ +406 static class WriteState { +407 // Set while a memstore flush is happening. +408 volatile boolean flushing = false; +409 // Set when a flush has been requested. +410 volatile boolean flushRequested = false; +411 // Number of compactions running. +412 AtomicInteger compacting = new AtomicInteger(0); +413 // Gets set in close. If set, cannot compact or flush again. +414 volatile boolean writesEnabled = true; +415 // Set if region is read-only +416 volatile boolean readOnly = false; +417 // whether the reads are enabled. This is different than readOnly, because readOnly is +418 // static in the lifetime of the region, while readsEnabled is dynamic +419 volatile boolean readsEnabled = true; +420 +421 /** +422 * Set flags that make this region read-only. +423 * +424 * @param onOff flip value for region r/o setting +425 */ +426 synchronized void setReadOnly(final boolean onOff) { +427 this.writesEnabled = !onOff; +428 this.readOnly = onOff; +429 } 430 -431 /** -432 * Objects from this class are created when flushing to describe all the different states that -433 * that method ends up in. The Result enum describes those states. The sequence id should only -434 * be specified if the flush was successful, and the failure message should only be specified -435 * if it didn't flush. -436 */ -437 public static class FlushResultImpl implements FlushResult { -438 final Result result; -439 final String failureReason; -440 final long flushSequenceId; -441 final boolean wroteFlushWalMarker; +431 boolean isReadOnly() { +432 return this.readOnly; +433 } +434 +435 boolean isFlushRequested() { +436 return this.flushRequested; +437 } +438 +439 void setReadsEnabled(boolean readsEnabled) { +440 this.readsEnabled = readsEnabled; +441 } 442 -443 /** -444 * Convenience constructor to use when the flush is successful, the failure message is set to -445 * null. -446 * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED. -447 * @param flushSequenceId Generated sequence id that comes right after the edits in the -448 * memstores. -449 */ -450 FlushResultImpl(Result result, long flushSequenceId) { -451 this(result, flushSequenceId, null, false); -452 assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result -453 .FLUSHED_COMPACTION_NEEDED; -454 } -455 -456 /** -457 * Convenience constructor to use when we cannot flush. -458 * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH. -459 * @param failureReason Reason why we couldn't flush. -460 */ -461 FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) { -462 this(result, -1, failureReason, wroteFlushMarker); -463 assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH; -464 } -465 -466 /** -467 * Constructor with all the parameters. -468 * @param result Any of the Result. -469 * @param flushSequenceId Generated sequence id if the memstores were flushed else -1. -470 * @param failureReason Reason why we couldn't flush, or null. -471 */ -472 FlushResultImpl(Result result, long flushSequenceId, String failureReason, -473 boolean wroteFlushMarker) { -474 this.result = result; -475 this.flushSequenceId = flushSequenceId; -476 this.failureReason = failureReason; -477 this.wroteFlushWalMarker = wroteFlushMarker; -478 } -479 -480 /** -481 * Convenience method, the equivalent of checking if result is -482 * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED. -483 * @return true if the memstores were flushed, else false. -484 */ -485 @Override -486 public boolean isFlushSucceeded() { -487 return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result -488 .FLUSHED_COMPACTION_NEEDED; -489 } -490 -491 /** -492 * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED. -493 * @return True if the flush requested a compaction, else false (doesn't even mean it flushed). -494 */ -495 @Override -496 public boolean isCompactionNeeded() { -497 return result == Result.FLUSHED_COMPACTION_NEEDED; -498 } -499 -500 @Override -501 public String toString() { -502 return new StringBuilder() -503 .append("flush result:").append(result).append(", ") -504 .append("failureReason:").append(failureReason).append(",") -505 .append("flush seq id").append(flushSequenceId).toString(); -506 } -507 -508 @Override -509 public Result getResult() { -510 return result; -511 } -512 } -513 -514 /** A result object from prepare flush cache stage */ -515 @VisibleForTesting -516 static class PrepareFlushResult { -517 final FlushResultImpl result; // indicating a failure result from prepare -518 final TreeMap<byte[], StoreFlushContext> storeFlushCtxs; -519 final TreeMap<byte[], List<Path>> committedFiles; -520 final TreeMap<byte[], MemStoreSize> storeFlushableSize; -521 final long startTime; -522 final long flushOpSeqId; -523 final long flushedSeqId; -524 final MemStoreSize totalFlushableSize; -525 -526 /** Constructs an early exit case */ -527 PrepareFlushResult(FlushResultImpl result, long flushSeqId) { -528 this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, new MemStoreSize()); -529 } -530 -531 /** Constructs a successful prepare flush result */ -532 PrepareFlushResult( -533 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, -534 TreeMap<byte[], List<Path>> committedFiles, -535 TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId, -536 long flushedSeqId, MemStoreSize totalFlushableSize) { -537 this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime, -538 flushSeqId, flushedSeqId, totalFlushableSize); -539 } -540 -541 private PrepareFlushResult( -542 FlushResultImpl result, -543 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, -544 TreeMap<byte[], List<Path>> committedFiles, -545 TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId, -546 long flushedSeqId, MemStoreSize totalFlushableSize) { -547 this.result = result; -548 this.storeFlushCtxs = storeFlushCtxs; -549 this.committedFiles = committedFiles; -550 this.storeFlushableSize = storeFlushableSize; -551 this.startTime = startTime; -552 this.flushOpSeqId = flushSeqId; -553 this.flushedSeqId = flushedSeqId; -554 this.totalFlushableSize = totalFlushableSize; +443 static final long HEAP_SIZE = ClassSize.align( +444 ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN); +445 } +446 +447 /** +448 * Objects from this class are created when flushing to describe all the different states that +449 * that method ends up in. The Result enum describes those states. The sequence id should only +450 * be specified if the flush was successful, and the failure message should only be specified +451 * if it didn't flush. +452 */ +453 public static class FlushResultImpl implements FlushResult { +454 final Result result; +455 final String failureReason; +456 final long flushSequenceId; +457 final boolean wroteFlushWalMarker; +458 +459 /** +460 * Convenience constructor to use when the flush is successful, the failure message is set to +461 * null. +462 * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED. +463 * @param flushSequenceId Generated sequence id that comes right after the edits in the +464 * memstores. +465 */ +466 FlushResultImpl(Result result, long flushSequenceId) { +467 this(result, flushSequenceId, null, false); +468 assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result +469 .FLUSHED_COMPACTION_NEEDED; +470 } +471 +472 /** +473 * Convenience constructor to use when we cannot flush. +474 * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH. +475 * @param failureReason Reason why we couldn't flush. +476 */ +477 FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) { +478 this(result, -1, failureReason, wroteFlushMarker); +479 assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH; +480 } +481 +482 /** +483 * Constructor with all the parameters. +484 * @param result Any of the Result. +485 * @param flushSequenceId Generated sequence id if the memstores were flushed else -1. +486 * @param failureReason Reason why we couldn't flush, or null. +487 */ +488 FlushResultImpl(Result result, long flushSequenceId, String failureReason, +489 boolean wroteFlushMarker) { +490 this.result = result; +491 this.flushSequenceId = flushSequenceId; +492