Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1D09D200D02 for ; Sat, 23 Sep 2017 17:13:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 1B47B1609E8; Sat, 23 Sep 2017 15:13:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C7B87160BD7 for ; Sat, 23 Sep 2017 17:13:41 +0200 (CEST) Received: (qmail 59803 invoked by uid 500); 23 Sep 2017 15:13:39 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 59184 invoked by uid 99); 23 Sep 2017 15:13:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 Sep 2017 15:13:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC23EF5AE8; Sat, 23 Sep 2017 15:13:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 23 Sep 2017 15:13:51 -0000 Message-Id: <5777c26bde5f44b5afe38c9fdbd4e363@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/30] hbase-site git commit: Published site at . archived-at: Sat, 23 Sep 2017 15:13:45 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/dc5c2985/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html index 6848d28..69caaf6 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html @@ -212,7941 +212,7899 @@ 204 public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = 205 "hbase.hregion.scan.loadColumnFamiliesOnDemand"; 206 -207 public static final String HREGION_UNASSIGN_FOR_FNFE = "hbase.hregion.unassign.for.fnfe"; -208 public static final boolean DEFAULT_HREGION_UNASSIGN_FOR_FNFE = true; +207 public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; +208 public static final int DEFAULT_MAX_CELL_SIZE = 10485760; 209 -210 public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; -211 public static final int DEFAULT_MAX_CELL_SIZE = 10485760; -212 -213 /** -214 * This is the global default value for durability. All tables/mutations not -215 * defining a durability or using USE_DEFAULT will default to this value. -216 */ -217 private static final Durability DEFAULT_DURABILITY = Durability.SYNC_WAL; -218 -219 final AtomicBoolean closed = new AtomicBoolean(false); -220 -221 /* Closing can take some time; use the closing flag if there is stuff we don't -222 * want to do while in closing state; e.g. like offer this region up to the -223 * master as a region to close if the carrying regionserver is overloaded. -224 * Once set, it is never cleared. -225 */ -226 final AtomicBoolean closing = new AtomicBoolean(false); -227 -228 /** -229 * The max sequence id of flushed data on this region. There is no edit in memory that is -230 * less that this sequence id. -231 */ -232 private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM; -233 -234 /** -235 * Record the sequence id of last flush operation. Can be in advance of -236 * {@link #maxFlushedSeqId} when flushing a single column family. In this case, -237 * {@link #maxFlushedSeqId} will be older than the oldest edit in memory. -238 */ -239 private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM; -240 -241 /** -242 * The sequence id of the last replayed open region event from the primary region. This is used -243 * to skip entries before this due to the possibility of replay edits coming out of order from -244 * replication. -245 */ -246 protected volatile long lastReplayedOpenRegionSeqId = -1L; -247 protected volatile long lastReplayedCompactionSeqId = -1L; -248 -249 ////////////////////////////////////////////////////////////////////////////// -250 // Members -251 ////////////////////////////////////////////////////////////////////////////// -252 -253 // map from a locked row to the context for that lock including: -254 // - CountDownLatch for threads waiting on that row -255 // - the thread that owns the lock (allow reentrancy) -256 // - reference count of (reentrant) locks held by the thread -257 // - the row itself -258 private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows = -259 new ConcurrentHashMap<>(); +210 /** +211 * This is the global default value for durability. All tables/mutations not +212 * defining a durability or using USE_DEFAULT will default to this value. +213 */ +214 private static final Durability DEFAULT_DURABILITY = Durability.SYNC_WAL; +215 +216 final AtomicBoolean closed = new AtomicBoolean(false); +217 +218 /* Closing can take some time; use the closing flag if there is stuff we don't +219 * want to do while in closing state; e.g. like offer this region up to the +220 * master as a region to close if the carrying regionserver is overloaded. +221 * Once set, it is never cleared. +222 */ +223 final AtomicBoolean closing = new AtomicBoolean(false); +224 +225 /** +226 * The max sequence id of flushed data on this region. There is no edit in memory that is +227 * less that this sequence id. +228 */ +229 private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM; +230 +231 /** +232 * Record the sequence id of last flush operation. Can be in advance of +233 * {@link #maxFlushedSeqId} when flushing a single column family. In this case, +234 * {@link #maxFlushedSeqId} will be older than the oldest edit in memory. +235 */ +236 private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM; +237 +238 /** +239 * The sequence id of the last replayed open region event from the primary region. This is used +240 * to skip entries before this due to the possibility of replay edits coming out of order from +241 * replication. +242 */ +243 protected volatile long lastReplayedOpenRegionSeqId = -1L; +244 protected volatile long lastReplayedCompactionSeqId = -1L; +245 +246 ////////////////////////////////////////////////////////////////////////////// +247 // Members +248 ////////////////////////////////////////////////////////////////////////////// +249 +250 // map from a locked row to the context for that lock including: +251 // - CountDownLatch for threads waiting on that row +252 // - the thread that owns the lock (allow reentrancy) +253 // - reference count of (reentrant) locks held by the thread +254 // - the row itself +255 private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows = +256 new ConcurrentHashMap<>(); +257 +258 protected final Map<byte[], HStore> stores = +259 new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR); 260 -261 protected final Map<byte[], HStore> stores = -262 new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR); +261 // TODO: account for each registered handler in HeapSize computation +262 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); 263 -264 // TODO: account for each registered handler in HeapSize computation -265 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); +264 private final AtomicLong memstoreDataSize = new AtomicLong(0);// Track data size in all memstores +265 private final RegionServicesForStores regionServicesForStores = new RegionServicesForStores(this); 266 -267 private final AtomicLong memstoreDataSize = new AtomicLong(0);// Track data size in all memstores -268 private final RegionServicesForStores regionServicesForStores = new RegionServicesForStores(this); -269 -270 // Debug possible data loss due to WAL off -271 final LongAdder numMutationsWithoutWAL = new LongAdder(); -272 final LongAdder dataInMemoryWithoutWAL = new LongAdder(); -273 -274 // Debug why CAS operations are taking a while. -275 final LongAdder checkAndMutateChecksPassed = new LongAdder(); -276 final LongAdder checkAndMutateChecksFailed = new LongAdder(); -277 -278 // Number of requests -279 final LongAdder readRequestsCount = new LongAdder(); -280 final LongAdder filteredReadRequestsCount = new LongAdder(); -281 final LongAdder writeRequestsCount = new LongAdder(); +267 // Debug possible data loss due to WAL off +268 final LongAdder numMutationsWithoutWAL = new LongAdder(); +269 final LongAdder dataInMemoryWithoutWAL = new LongAdder(); +270 +271 // Debug why CAS operations are taking a while. +272 final LongAdder checkAndMutateChecksPassed = new LongAdder(); +273 final LongAdder checkAndMutateChecksFailed = new LongAdder(); +274 +275 // Number of requests +276 final LongAdder readRequestsCount = new LongAdder(); +277 final LongAdder filteredReadRequestsCount = new LongAdder(); +278 final LongAdder writeRequestsCount = new LongAdder(); +279 +280 // Number of requests blocked by memstore size. +281 private final LongAdder blockedRequestsCount = new LongAdder(); 282 -283 // Number of requests blocked by memstore size. -284 private final LongAdder blockedRequestsCount = new LongAdder(); -285 -286 // Compaction LongAdders -287 final LongAdder compactionsFinished = new LongAdder(); -288 final LongAdder compactionsFailed = new LongAdder(); -289 final LongAdder compactionNumFilesCompacted = new LongAdder(); -290 final LongAdder compactionNumBytesCompacted = new LongAdder(); -291 final LongAdder compactionsQueued = new LongAdder(); -292 final LongAdder flushesQueued = new LongAdder(); -293 -294 private final WAL wal; -295 private final HRegionFileSystem fs; -296 protected final Configuration conf; -297 private final Configuration baseConf; -298 private final int rowLockWaitDuration; -299 static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000; -300 -301 // The internal wait duration to acquire a lock before read/update -302 // from the region. It is not per row. The purpose of this wait time -303 // is to avoid waiting a long time while the region is busy, so that -304 // we can release the IPC handler soon enough to improve the -305 // availability of the region server. It can be adjusted by -306 // tuning configuration "hbase.busy.wait.duration". -307 final long busyWaitDuration; -308 static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT; -309 -310 // If updating multiple rows in one call, wait longer, -311 // i.e. waiting for busyWaitDuration * # of rows. However, -312 // we can limit the max multiplier. -313 final int maxBusyWaitMultiplier; -314 -315 // Max busy wait duration. There is no point to wait longer than the RPC -316 // purge timeout, when a RPC call will be terminated by the RPC engine. -317 final long maxBusyWaitDuration; -318 -319 // Max cell size. If nonzero, the maximum allowed size for any given cell -320 // in bytes -321 final long maxCellSize; -322 -323 // negative number indicates infinite timeout -324 static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L; -325 final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool(); -326 -327 private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints; -328 -329 /** -330 * The sequence ID that was enLongAddered when this region was opened. -331 */ -332 private long openSeqNum = HConstants.NO_SEQNUM; -333 -334 /** -335 * The default setting for whether to enable on-demand CF loading for -336 * scan requests to this region. Requests can override it. -337 */ -338 private boolean isLoadingCfsOnDemandDefault = false; +283 // Compaction LongAdders +284 final LongAdder compactionsFinished = new LongAdder(); +285 final LongAdder compactionsFailed = new LongAdder(); +286 final LongAdder compactionNumFilesCompacted = new LongAdder(); +287 final LongAdder compactionNumBytesCompacted = new LongAdder(); +288 final LongAdder compactionsQueued = new LongAdder(); +289 final LongAdder flushesQueued = new LongAdder(); +290 +291 private final WAL wal; +292 private final HRegionFileSystem fs; +293 protected final Configuration conf; +294 private final Configuration baseConf; +295 private final int rowLockWaitDuration; +296 static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000; +297 +298 // The internal wait duration to acquire a lock before read/update +299 // from the region. It is not per row. The purpose of this wait time +300 // is to avoid waiting a long time while the region is busy, so that +301 // we can release the IPC handler soon enough to improve the +302 // availability of the region server. It can be adjusted by +303 // tuning configuration "hbase.busy.wait.duration". +304 final long busyWaitDuration; +305 static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT; +306 +307 // If updating multiple rows in one call, wait longer, +308 // i.e. waiting for busyWaitDuration * # of rows. However, +309 // we can limit the max multiplier. +310 final int maxBusyWaitMultiplier; +311 +312 // Max busy wait duration. There is no point to wait longer than the RPC +313 // purge timeout, when a RPC call will be terminated by the RPC engine. +314 final long maxBusyWaitDuration; +315 +316 // Max cell size. If nonzero, the maximum allowed size for any given cell +317 // in bytes +318 final long maxCellSize; +319 +320 // negative number indicates infinite timeout +321 static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L; +322 final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool(); +323 +324 private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints; +325 +326 /** +327 * The sequence ID that was enLongAddered when this region was opened. +328 */ +329 private long openSeqNum = HConstants.NO_SEQNUM; +330 +331 /** +332 * The default setting for whether to enable on-demand CF loading for +333 * scan requests to this region. Requests can override it. +334 */ +335 private boolean isLoadingCfsOnDemandDefault = false; +336 +337 private final AtomicInteger majorInProgress = new AtomicInteger(0); +338 private final AtomicInteger minorInProgress = new AtomicInteger(0); 339 -340 private final AtomicInteger majorInProgress = new AtomicInteger(0); -341 private final AtomicInteger minorInProgress = new AtomicInteger(0); -342 -343 // -344 // Context: During replay we want to ensure that we do not lose any data. So, we -345 // have to be conservative in how we replay wals. For each store, we calculate -346 // the maxSeqId up to which the store was flushed. And, skip the edits which -347 // are equal to or lower than maxSeqId for each store. -348 // The following map is populated when opening the region -349 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); +340 // +341 // Context: During replay we want to ensure that we do not lose any data. So, we +342 // have to be conservative in how we replay wals. For each store, we calculate +343 // the maxSeqId up to which the store was flushed. And, skip the edits which +344 // are equal to or lower than maxSeqId for each store. +345 // The following map is populated when opening the region +346 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); +347 +348 /** Saved state from replaying prepare flush cache */ +349 private PrepareFlushResult prepareFlushResult = null; 350 -351 /** Saved state from replaying prepare flush cache */ -352 private PrepareFlushResult prepareFlushResult = null; -353 -354 /** -355 * Config setting for whether to allow writes when a region is in recovering or not. -356 */ -357 private boolean disallowWritesInRecovering = false; +351 /** +352 * Config setting for whether to allow writes when a region is in recovering or not. +353 */ +354 private boolean disallowWritesInRecovering = false; +355 +356 // When a region is in recovering state, it can only accept writes not reads +357 private volatile boolean recovering = false; 358 -359 // When a region is in recovering state, it can only accept writes not reads -360 private volatile boolean recovering = false; -361 -362 private volatile Optional<ConfigurationManager> configurationManager; +359 private volatile Optional<ConfigurationManager> configurationManager; +360 +361 // Used for testing. +362 private volatile Long timeoutForWriteLock = null; 363 -364 // Used for testing. -365 private volatile Long timeoutForWriteLock = null; -366 -367 /** -368 * @return The smallest mvcc readPoint across all the scanners in this -369 * region. Writes older than this readPoint, are included in every -370 * read operation. -371 */ -372 public long getSmallestReadPoint() { -373 long minimumReadPoint; -374 // We need to ensure that while we are calculating the smallestReadPoint -375 // no new RegionScanners can grab a readPoint that we are unaware of. -376 // We achieve this by synchronizing on the scannerReadPoints object. -377 synchronized (scannerReadPoints) { -378 minimumReadPoint = mvcc.getReadPoint(); -379 for (Long readPoint : this.scannerReadPoints.values()) { -380 if (readPoint < minimumReadPoint) { -381 minimumReadPoint = readPoint; -382 } -383 } -384 } -385 return minimumReadPoint; -386 } -387 -388 /* -389 * Data structure of write state flags used coordinating flushes, -390 * compactions and closes. -391 */ -392 static class WriteState { -393 // Set while a memstore flush is happening. -394 volatile boolean flushing = false; -395 // Set when a flush has been requested. -396 volatile boolean flushRequested = false; -397 // Number of compactions running. -398 AtomicInteger compacting = new AtomicInteger(0); -399 // Gets set in close. If set, cannot compact or flush again. -400 volatile boolean writesEnabled = true; -401 // Set if region is read-only -402 volatile boolean readOnly = false; -403 // whether the reads are enabled. This is different than readOnly, because readOnly is -404 // static in the lifetime of the region, while readsEnabled is dynamic -405 volatile boolean readsEnabled = true; -406 -407 /** -408 * Set flags that make this region read-only. -409 * -410 * @param onOff flip value for region r/o setting -411 */ -412 synchronized void setReadOnly(final boolean onOff) { -413 this.writesEnabled = !onOff; -414 this.readOnly = onOff; -415 } -416 -417 boolean isReadOnly() { -418 return this.readOnly; -419 } -420 -421 boolean isFlushRequested() { -422 return this.flushRequested; -423 } -424 -425 void setReadsEnabled(boolean readsEnabled) { -426 this.readsEnabled = readsEnabled; -427 } -428 -429 static final long HEAP_SIZE = ClassSize.align( -430 ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN); -431 } -432 -433 /** -434 * Objects from this class are created when flushing to describe all the different states that -435 * that method ends up in. The Result enum describes those states. The sequence id should only -436 * be specified if the flush was successful, and the failure message should only be specified -437 * if it didn't flush. -438 */ -439 public static class FlushResultImpl implements FlushResult { -440 final Result result; -441 final String failureReason; -442 final long flushSequenceId; -443 final boolean wroteFlushWalMarker; -444 -445 /** -446 * Convenience constructor to use when the flush is successful, the failure message is set to -447 * null. -448 * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED. -449 * @param flushSequenceId Generated sequence id that comes right after the edits in the -450 * memstores. -451 */ -452 FlushResultImpl(Result result, long flushSequenceId) { -453 this(result, flushSequenceId, null, false); -454 assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result -455 .FLUSHED_COMPACTION_NEEDED; -456 } -457 -458 /** -459 * Convenience constructor to use when we cannot flush. -460 * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH. -461 * @param failureReason Reason why we couldn't flush. -462 */ -463 FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) { -464 this(result, -1, failureReason, wroteFlushMarker); -465 assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH; -466 } -467 -468 /** -469 * Constructor with all the parameters. -470 * @param result Any of the Result. -471 * @param flushSequenceId Generated sequence id if the memstores were flushed else -1. -472 * @param failureReason Reason why we couldn't flush, or null. -473 */ -474 FlushResultImpl(Result result, long flushSequenceId, String failureReason, -475 boolean wroteFlushMarker) { -476 this.result = result; -477 this.flushSequenceId = flushSequenceId; -478 this.failureReason = failureReason; -479 this.wroteFlushWalMarker = wroteFlushMarker; -480 } -481 -482 /** -483 * Convenience method, the equivalent of checking if result is -484 * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED. -485 * @return true if the memstores were flushed, else false. -486 */ -487 @Override -488 public boolean isFlushSucceeded() { -489 return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result -490 .FLUSHED_COMPACTION_NEEDED; -491 } -492 -493 /** -494 * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED. -495 * @return True if the flush requested a compaction, else false (doesn't even mean it flushed). -496 */ -497 @Override -498 public boolean isCompactionNeeded() { -499 return result == Result.FLUSHED_COMPACTION_NEEDED; -500 } -501 -502 @Override -503 public String toString() { -504 return new StringBuilder() -505 .append("flush result:").append(result).append(", ") -506 .append("failureReason:").append(failureReason).append(",") -507 .append("flush seq id").append(flushSequenceId).toString(); -508 } -509 -510 @Override -511 public Result getResult() { -512 return result; -513 } -514 } -515 -516 /** A result object from prepare flush cache stage */ -517 @VisibleForTesting -518 static class PrepareFlushResult { -519 final FlushResultImpl result; // indicating a failure result from prepare -520 final TreeMap<byte[], StoreFlushContext> storeFlushCtxs; -521 final TreeMap<byte[], List<Path>> committedFiles; -522 final TreeMap<byte[], MemstoreSize> storeFlushableSize; -523 final long startTime; -524 final long flushOpSeqId; -525 final long flushedSeqId; -526 final MemstoreSize totalFlushableSize; -527 -528 /** Constructs an early exit case */ -529 PrepareFlushResult(FlushResultImpl result, long flushSeqId) { -530 this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, new MemstoreSize()); -531 } -532 -533 /** Constructs a successful prepare flush result */ -534 PrepareFlushResult( -535 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, -536 TreeMap<byte[], List<Path>> committedFiles, -537 TreeMap<byte[], MemstoreSize> storeFlushableSize, long startTime, long flushSeqId, -538 long flushedSeqId, MemstoreSize totalFlushableSize) { -539 this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime, -540 flushSeqId, flushedSeqId, totalFlushableSize); -541 } -542 -543 private PrepareFlushResult( -544 FlushResultImpl result, -545 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, -546 TreeMap<byte[], List<Path>> committedFiles, -547 TreeMap<byte[], MemstoreSize> storeFlushableSize, long startTime, long flushSeqId, -548 long flushedSeqId, MemstoreSize totalFlushableSize) { -549 this.result = result; -550 this.storeFlushCtxs = storeFlushCtxs; -551 this.committedFiles = committedFiles; -552 this.storeFlushableSize = storeFlushableSize; -553 this.startTime = startTime; -554 this.flushOpSeqId = flushSeqId; -555 this.flushedSeqId = flushedSeqId; -556 this.totalFlushableSize = totalFlushableSize; -557 } -558 -559 public FlushResult getResult() { -560 return this.result; -561 } -562 } -563 -564 /** -565 * A class that tracks exceptions that have been observed in one batch. Not thread safe. -566 */ -567 static class ObservedExceptionsInBatch { -568 private boolean wrongRegion = false; -569 private boolean failedSanityCheck = false; -570 private boolean wrongFamily = false; -571 -572 /** -573 * @return If a {@link WrongRegionException} has been observed. -574 */ -575 boolean hasSeenWrongRegion() { -576 return wrongRegion; -577 } -578 -579 /** -580 * Records that a {@link WrongRegionException} has been observed. -581 */ -582 void sawWrongRegion() { -583 wrongRegion = true; -584 } -585 -586 /** -587 * @return If a {@link FailedSanityCheckException} has been observed. -588 */ -589 boolean hasSeenFailedSanityCheck() { -590 return failedSanityCheck; -591 } -592 -593 /** -594 * Records that a {@link FailedSanityCheckException} has been observed. -595 */ -596 void sawFailedSanityCheck() { -597 failedSanityCheck = true; -598 } -599 -600 /** -601 * @return If a {@link NoSuchColumnFamilyException} has been observed. -602 */ -603 boolean hasSeenNoSuchFamily() { -604 return wrongFamily; -605 } -606 -607 /** -608 * Records that a {@link NoSuchColumnFamilyException} has been observed. -609 */ -610 void sawNoSuchFamily() { -611 wrongFamily = true; -612 } -613 } -614 -615 final WriteState writestate = new WriteState(); -616 -617 long memstoreFlushSize; -618 final long timestampSlop; -619 final long rowProcessorTimeout; +364 /** +365 * @return The smallest mvcc readPoint across all the scanners in this +366 * region. Writes older than this readPoint, are included in every +367 * read operation. +368 */ +369 public long getSmallestReadPoint() { +370 long minimumReadPoint; +371 // We need to ensure that while we are calculating the smallestReadPoint +372 // no new RegionScanners can grab a readPoint that we are unaware of. +373 // We achieve this by synchronizing on the scannerReadPoints object. +374 synchronized (scannerReadPoints) { +375 minimumReadPoint = mvcc.getReadPoint(); +376 for (Long readPoint : this.scannerReadPoints.values()) { +377 if (readPoint < minimumReadPoint) { +378 minimumReadPoint = readPoint; +379 } +380 } +381 } +382 return minimumReadPoint; +383 } +384 +385 /* +386 * Data structure of write state flags used coordinating flushes, +387 * compactions and closes. +388 */ +389 static class WriteState { +390 // Set while a memstore flush is happening. +391 volatile boolean flushing = false; +392 // Set when a flush has been requested. +393 volatile boolean flushRequested = false; +394 // Number of compactions running. +395 AtomicInteger compacting = new AtomicInteger(0); +396 // Gets set in close. If set, cannot compact or flush again. +397 volatile boolean writesEnabled = true; +398 // Set if region is read-only +399 volatile boolean readOnly = false; +400 // whether the reads are enabled. This is different than readOnly, because readOnly is +401 // static in the lifetime of the region, while readsEnabled is dynamic +402 volatile boolean readsEnabled = true; +403 +404 /** +405 * Set flags that make this region read-only. +406 * +407 * @param onOff flip value for region r/o setting +408 */ +409 synchronized void setReadOnly(final boolean onOff) { +410 this.writesEnabled = !onOff; +411 this.readOnly = onOff; +412 } +413 +414 boolean isReadOnly() { +415 return this.readOnly; +416 } +417 +418 boolean isFlushRequested() { +419 return this.flushRequested; +420 } +421 +422 void setReadsEnabled(boolean readsEnabled) { +423 this.readsEnabled = readsEnabled; +424 } +425 +426 static final long HEAP_SIZE = ClassSize.align( +427 ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN); +428 } +429 +430 /** +431 * Objects from this class are created when flushing to describe all the different states that +432 * that method ends up in. The Result enum describes those states. The sequence id should only +433 * be specified if the flush was successful, and the failure message should only be specified +434 * if it didn't flush. +435 */ +436 public static class FlushResultImpl implements FlushResult { +437 final Result result; +438 final String failureReason; +439 final long flushSequenceId; +440 final boolean wroteFlushWalMarker; +441 +442 /** +443 * Convenience constructor to use when the flush is successful, the failure message is set to +444 * null. +445 * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED. +446 * @param flushSequenceId Generated sequence id that comes right after the edits in the +447 * memstores. +448 */ +449 FlushResultImpl(Result result, long flushSequenceId) { +450 this(result, flushSequenceId, null, false); +451 assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result +452 .FLUSHED_COMPACTION_NEEDED; +453 } +454 +455 /** +456 * Convenience constructor to use when we cannot flush. +457 * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH. +458 * @param failureReason Reason why we couldn't flush. +459 */ +460 FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) { +461 this(result, -1, failureReason, wroteFlushMarker); +462 assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH; +463 } +464 +465 /** +466 * Constructor with all the parameters. +467 * @param result Any of the Result. +468 * @param flushSequenceId Generated sequence id if the memstores were flushed else -1. +469 * @param failureReason Reason why we couldn't flush, or null. +470 */ +471 FlushResultImpl(Result result, long flushSequenceId, String failureReason, +472 boolean wroteFlushMarker) { +473 this.result = result; +474 this.flushSequenceId = flushSequenceId; +475 this.failureReason = failureReason; +476 this.wroteFlushWalMarker = wroteFlushMarker; +477 } +478 +479 /** +480 * Convenience method, the equivalent of checking if result is +481 * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED. +482 * @return true if the memstores were flushed, else false. +483 */ +484 @Override +485 public boolean isFlushSucceeded() { +486 return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result +487 .FLUSHED_COMPACTION_NEEDED; +488 } +489 +490 /** +491 * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED. +492 * @return True if the flush requested a compaction, else false (doesn't even mean it flushed). +493 */ +494 @Override +495 public boolean isCompactionNeeded() { +496 return result == Result.FLUSHED_COMPACTION_NEEDED; +497 } +498 +499 @Override +500 public String toString() { +501 return new StringBuilder() +502 .append("flush result:").append(result).append(", ") +503 .append("failureReason:").append(failureReason).append(",") +504 .append("flush seq id").append(flushSequenceId).toString(); +505 } +506 +507 @Override +508 public Result getResult() { +509 return result; +510 } +511 } +512 +513 /** A result object from prepare flush cache stage */ +514 @VisibleForTesting +515 static class PrepareFlushResult { +516 final FlushResultImpl result; // indicating a failure result from prepare +517 final TreeMap<byte[], StoreFlushContext> storeFlushCtxs; +518 final TreeMap<byte[], List<Path>> committedFiles; +519 final TreeMap<byte[], MemstoreSize> storeFlushableSize; +520 final long startTime; +521 final long flushOpSeqId; +522 final long flushedSeqId; +523 final MemstoreSize totalFlushableSize; +524 +525 /** Constructs an early exit case */ +526 PrepareFlushResult(FlushResultImpl result, long flushSeqId) { +527 this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, new MemstoreSize()); +528 } +529 +530 /** Constructs a successful prepare flush result */ +531 PrepareFlushResult( +532 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, +533 TreeMap<byte[], List<Path>> committedFiles, +534 TreeMap<byte[], MemstoreSize> storeFlushableSize, long startTime, long flushSeqId, +535 long flushedSeqId, MemstoreSize totalFlushableSize) { +536 this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime, +537 flushSeqId, flushedSeqId, totalFlushableSize); +538 } +539 +540 private PrepareFlushResult( +541 FlushResultImpl result, +542 TreeMap<byte[], StoreFlushContext> storeFlushCtxs, +543 TreeMap<byte[], List<Path>> committedFiles, +544 TreeMap<byte[], MemstoreSize> storeFlushableSize, long startTime, long flushSeqId, +545 long flushedSeqId, MemstoreSize totalFlushableSize) { +546 this.result = result; +547 this.storeFlushCtxs = storeFlushCtxs; +548 this.committedFiles = committedFiles; +549 this.storeFlushableSize = storeFlushableSize; +550 this.startTime = startTime; +551 this.flushOpSeqId = flushSeqId; +552 this.flushedSeqId = flushedSeqId; +553 this.totalFlushableSize = totalFlushableSize; +554 } +555 +556 public FlushResult getResult() { +557 return this.result; +558 } +559 } +560 +561 /** +562 * A class that tracks exceptions that have been observed in one batch. Not thread safe. +563 */ +564 static class ObservedExceptionsInBatch { +565 private boolean wrongRegion = false; +566 private boolean failedSanityCheck = false; +567 private boolean wrongFamily = false; +568 +569 /** +570 * @return If a {@link WrongRegionException} has been observed. +571 */ +572 boolean hasSeenWrongRegion() { +573 return wrongRegion; +574 } +575 +576 /** +577 * Records that a {@link WrongRegionException} has been observed. +578 */ +579 void sawWrongRegion() { +580 wrongRegion = true; +581 } +582 +583 /** +584 * @return If a {@link FailedSanityCheckException} has been observed. +585 */ +586 boolean hasSeenFailedSanityCheck() { +587 return failedSanityCheck; +588 } +589 +590 /** +591 * Records that a {@link FailedSanityCheckException} has been observed. +592 */ +593 void sawFailedSanityCheck() { +594 failedSanityCheck = true; +595 } +596 +597 /** +598 * @return If a {@link NoSuchColumnFamilyException} has been observed. +599 */ +600 boolean hasSeenNoSuchFamily() { +601 return wrongFamily; +602 } +603 +604 /** +605 * Records that a {@link NoSuchColumnFamilyException} has been observed. +606 */ +607 void sawNoSuchFamily() { +608 wrongFamily = true; +609 } +610 } +611 +612 final WriteState writestate = new WriteState(); +613 +614 long memstoreFlushSize; +615 final long timestampSlop; +616 final long rowProcessorTimeout; +617 +618 // Last flush time for each Store. Useful when we are flushing for each column +619 private final ConcurrentMap<HStore, Long> lastStoreFlushTimeMap = new ConcurrentHashMap<>(); 620 -621 // Last flush time for each Store. Useful when we are flushing for each column -622 private final ConcurrentMap<HStore, Long> lastStoreFlushTimeMap = new ConcurrentHashMap<>(); -623 -624 final RegionServerServices rsServices; -625 private RegionServerAccounting rsAccounting; -626 private long flushCheckInterval; -627 // flushPerChanges is to prevent too many changes in memstore -628 private long flushPerChanges; -629 private long blockingMemStoreSize; -630 final long threadWakeFrequency; -631 // Used to guard closes -632 final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); -633 -634 // Stop updates lock -635 private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock(); -636 private boolean splitRequest; -637 private byte[] explicitSplitPoint = null; -638 -639 private final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); +621 final RegionServerServices rsServices; +622 private RegionServerAccounting rsAccounting; +623 private long flushCheckInterval; +624 // flushPerChanges is to prevent too many changes in memstore +625 private long flushPerChanges; +626 private long blockingMemStoreSize; +627 final long threadWakeFrequency; +628 // Used to guard closes +629 final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); +630 +631 // Stop updates lock +632 private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock(); +633 private boolean splitRequest; +634 private byte[] explicitSplitPoint = null; +635 +636 private final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); +637 +638 // Coprocessor host +639 private RegionCoprocessorHost coprocessorHost; 640 -641 // Coprocessor host -642 private RegionCoprocessorHost coprocessorHost; -643 -644 private TableDescriptor htableDescriptor = null; -645 private RegionSplitPolicy splitPolicy; -646 private FlushPolicy flushPolicy; -647 -648 private final MetricsRegion metricsRegion; -649 private final MetricsRegionWrapperImpl metricsRegionWrapper; -650 private final Durability durability; -651 private final boolean regionStatsEnabled; -652 // Stores the replication scope of the various column families of the table -653 // that has non-default scope -654 private final NavigableMap<byte[], Integer> replicationScope = new TreeMap<>( -655 Bytes.BYTES_COMPARATOR); -656 -657 // whether to unassign region if we hit FNFE -658 private final RegionUnassigner regionUnassigner; -659 /** -660 * HRegion constructor. This constructor should only be used for testing and -661 * extensions. Instances of HRegion should be instantiated with the -662 * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method. -663 * -664 * @param tableDir qualified path of directory where region should be located, -665 * usually the table directory. -666 * @param wal The WAL is the outbound log for any updates to the HRegion -667 * The wal file is a logfile from the previous execution that's -668 * custom-computed for this HRegion. The HRegionServer computes and sorts the -669 * appropriate wal info for this HRegion. If there is a previous wal file -670 * (implying that the HRegion has been written-to before), then read it from -671 * the supplied path. -672 * @param fs is the filesystem. -673