From commits-return-65335-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Fri Jan 12 16:31:50 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id EBBB71807AC for ; Fri, 12 Jan 2018 16:31:46 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id DB898160C30; Fri, 12 Jan 2018 15:31:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id DD345160C4F for ; Fri, 12 Jan 2018 16:31:43 +0100 (CET) Received: (qmail 24549 invoked by uid 500); 12 Jan 2018 15:31:39 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 21056 invoked by uid 99); 12 Jan 2018 15:31:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 12 Jan 2018 15:31:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D1730F3514; Fri, 12 Jan 2018 15:31:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Fri, 12 Jan 2018 15:32:02 -0000 Message-Id: <4acbaaf4986f41cd9960d7704664ac9e@git.apache.org> In-Reply-To: <50f4f80b288941cd871f64f9f78d188a@git.apache.org> References: <50f4f80b288941cd871f64f9f78d188a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [33/51] [partial] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/0b638133/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.PeriodicMemStoreFlusher.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.PeriodicMemStoreFlusher.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.PeriodicMemStoreFlusher.html index 163ade0..802fc2f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.PeriodicMemStoreFlusher.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.PeriodicMemStoreFlusher.html @@ -254,3512 +254,3505 @@ 246 protected MemStoreFlusher cacheFlusher; 247 248 protected HeapMemoryManager hMemManager; -249 protected CountDownLatch initLatch = null; -250 -251 /** -252 * Cluster connection to be shared by services. -253 * Initialized at server startup and closed when server shuts down. -254 * Clients must never close it explicitly. -255 */ -256 protected ClusterConnection clusterConnection; -257 -258 /* -259 * Long-living meta table locator, which is created when the server is started and stopped -260 * when server shuts down. References to this locator shall be used to perform according -261 * operations in EventHandlers. Primary reason for this decision is to make it mockable -262 * for tests. -263 */ -264 protected MetaTableLocator metaTableLocator; -265 -266 /** -267 * Go here to get table descriptors. -268 */ -269 protected TableDescriptors tableDescriptors; -270 -271 // Replication services. If no replication, this handler will be null. -272 protected ReplicationSourceService replicationSourceHandler; -273 protected ReplicationSinkService replicationSinkHandler; -274 -275 // Compactions -276 public CompactSplit compactSplitThread; -277 -278 /** -279 * Map of regions currently being served by this region server. Key is the -280 * encoded region name. All access should be synchronized. -281 */ -282 protected final Map<String, HRegion> onlineRegions = new ConcurrentHashMap<>(); -283 -284 /** -285 * Map of encoded region names to the DataNode locations they should be hosted on -286 * We store the value as InetSocketAddress since this is used only in HDFS -287 * API (create() that takes favored nodes as hints for placing file blocks). -288 * We could have used ServerName here as the value class, but we'd need to -289 * convert it to InetSocketAddress at some point before the HDFS API call, and -290 * it seems a bit weird to store ServerName since ServerName refers to RegionServers -291 * and here we really mean DataNode locations. -292 */ -293 protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap = -294 new ConcurrentHashMap<>(); -295 -296 // Leases -297 protected Leases leases; -298 -299 // Instance of the hbase executor executorService. -300 protected ExecutorService executorService; -301 -302 // If false, the file system has become unavailable -303 protected volatile boolean fsOk; -304 protected HFileSystem fs; -305 protected HFileSystem walFs; -306 -307 // Set when a report to the master comes back with a message asking us to -308 // shutdown. Also set by call to stop when debugging or running unit tests -309 // of HRegionServer in isolation. -310 private volatile boolean stopped = false; -311 -312 // Go down hard. Used if file system becomes unavailable and also in -313 // debugging and unit tests. -314 private volatile boolean abortRequested; -315 -316 ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<>(); -317 -318 // A state before we go into stopped state. At this stage we're closing user -319 // space regions. -320 private boolean stopping = false; -321 -322 volatile boolean killed = false; -323 -324 protected final Configuration conf; -325 -326 private Path rootDir; -327 private Path walRootDir; -328 -329 protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); -330 -331 final int numRetries; -332 protected final int threadWakeFrequency; -333 protected final int msgInterval; -334 -335 protected final int numRegionsToReport; -336 -337 // Stub to do region server status calls against the master. -338 private volatile RegionServerStatusService.BlockingInterface rssStub; -339 private volatile LockService.BlockingInterface lockStub; -340 // RPC client. Used to make the stub above that does region server status checking. -341 RpcClient rpcClient; -342 -343 private RpcRetryingCallerFactory rpcRetryingCallerFactory; -344 private RpcControllerFactory rpcControllerFactory; -345 -346 private UncaughtExceptionHandler uncaughtExceptionHandler; -347 -348 // Info server. Default access so can be used by unit tests. REGIONSERVER -349 // is name of the webapp and the attribute name used stuffing this instance -350 // into web context. -351 protected InfoServer infoServer; -352 private JvmPauseMonitor pauseMonitor; -353 -354 /** region server process name */ -355 public static final String REGIONSERVER = "regionserver"; -356 -357 MetricsRegionServer metricsRegionServer; -358 MetricsTable metricsTable; -359 private SpanReceiverHost spanReceiverHost; -360 -361 /** -362 * ChoreService used to schedule tasks that we want to run periodically -363 */ -364 private ChoreService choreService; -365 -366 /* -367 * Check for compactions requests. -368 */ -369 ScheduledChore compactionChecker; -370 -371 /* -372 * Check for flushes -373 */ -374 ScheduledChore periodicFlusher; -375 -376 protected volatile WALFactory walFactory; -377 -378 // WAL roller. log is protected rather than private to avoid -379 // eclipse warning when accessed by inner classes -380 protected LogRoller walRoller; -381 -382 // A thread which calls reportProcedureDone -383 private RemoteProcedureResultReporter procedureResultReporter; -384 -385 // flag set after we're done setting up server threads -386 final AtomicBoolean online = new AtomicBoolean(false); -387 -388 // zookeeper connection and watcher -389 protected ZKWatcher zooKeeper; -390 -391 // master address tracker -392 private MasterAddressTracker masterAddressTracker; -393 -394 // Cluster Status Tracker -395 protected ClusterStatusTracker clusterStatusTracker; -396 -397 // Log Splitting Worker -398 private SplitLogWorker splitLogWorker; -399 -400 // A sleeper that sleeps for msgInterval. -401 protected final Sleeper sleeper; -402 -403 private final int operationTimeout; -404 private final int shortOperationTimeout; -405 -406 private final RegionServerAccounting regionServerAccounting; -407 -408 // Cache configuration and block cache reference -409 protected CacheConfig cacheConfig; -410 // Cache configuration for mob -411 final MobCacheConfig mobCacheConfig; -412 -413 /** The health check chore. */ -414 private HealthCheckChore healthCheckChore; -415 -416 /** The nonce manager chore. */ -417 private ScheduledChore nonceManagerChore; -418 -419 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); -420 -421 /** -422 * The server name the Master sees us as. Its made from the hostname the -423 * master passes us, port, and server startcode. Gets set after registration -424 * against Master. -425 */ -426 protected ServerName serverName; -427 -428 /* -429 * hostname specified by hostname config -430 */ -431 protected String useThisHostnameInstead; -432 -433 // key to the config parameter of server hostname -434 // the specification of server hostname is optional. The hostname should be resolvable from -435 // both master and region server -436 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -437 final static String RS_HOSTNAME_KEY = "hbase.regionserver.hostname"; -438 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -439 protected final static String MASTER_HOSTNAME_KEY = "hbase.master.hostname"; -440 -441 // HBASE-18226: This config and hbase.regionserver.hostname are mutually exclusive. -442 // Exception will be thrown if both are used. -443 final static String RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY = -444 "hbase.regionserver.hostname.disable.master.reversedns"; -445 -446 /** -447 * This servers startcode. -448 */ -449 protected final long startcode; -450 -451 /** -452 * Unique identifier for the cluster we are a part of. -453 */ -454 private String clusterId; -455 -456 /** -457 * MX Bean for RegionServerInfo -458 */ -459 private ObjectName mxBean = null; -460 -461 /** -462 * Chore to clean periodically the moved region list -463 */ -464 private MovedRegionsCleaner movedRegionsCleaner; -465 -466 // chore for refreshing store files for secondary regions -467 private StorefileRefresherChore storefileRefresher; -468 -469 private RegionServerCoprocessorHost rsHost; -470 -471 private RegionServerProcedureManagerHost rspmHost; -472 -473 private RegionServerRpcQuotaManager rsQuotaManager; -474 private RegionServerSpaceQuotaManager rsSpaceQuotaManager; -475 -476 /** -477 * Nonce manager. Nonces are used to make operations like increment and append idempotent -478 * in the case where client doesn't receive the response from a successful operation and -479 * retries. We track the successful ops for some time via a nonce sent by client and handle -480 * duplicate operations (currently, by failing them; in future we might use MVCC to return -481 * result). Nonces are also recovered from WAL during, recovery; however, the caveats (from -482 * HBASE-3787) are: -483 * - WAL recovery is optimized, and under high load we won't read nearly nonce-timeout worth -484 * of past records. If we don't read the records, we don't read and recover the nonces. -485 * Some WALs within nonce-timeout at recovery may not even be present due to rolling/cleanup. -486 * - There's no WAL recovery during normal region move, so nonces will not be transfered. -487 * We can have separate additional "Nonce WAL". It will just contain bunch of numbers and -488 * won't be flushed on main path - because WAL itself also contains nonces, if we only flush -489 * it before memstore flush, for a given nonce we will either see it in the WAL (if it was -490 * never flushed to disk, it will be part of recovery), or we'll see it as part of the nonce -491 * log (or both occasionally, which doesn't matter). Nonce log file can be deleted after the -492 * latest nonce in it expired. It can also be recovered during move. -493 */ -494 final ServerNonceManager nonceManager; -495 -496 private UserProvider userProvider; -497 -498 protected final RSRpcServices rpcServices; -499 -500 protected CoordinatedStateManager csm; -501 -502 /** -503 * Configuration manager is used to register/deregister and notify the configuration observers -504 * when the regionserver is notified that there was a change in the on disk configs. -505 */ -506 protected final ConfigurationManager configurationManager; -507 -508 @VisibleForTesting -509 CompactedHFilesDischarger compactedFileDischarger; -510 -511 private volatile ThroughputController flushThroughputController; -512 -513 protected SecureBulkLoadManager secureBulkLoadManager; -514 -515 protected FileSystemUtilizationChore fsUtilizationChore; -516 -517 private final NettyEventLoopGroupConfig eventLoopGroupConfig; -518 -519 /** -520 * True if this RegionServer is coming up in a cluster where there is no Master; -521 * means it needs to just come up and make do without a Master to talk to: e.g. in test or -522 * HRegionServer is doing other than its usual duties: e.g. as an hollowed-out host whose only -523 * purpose is as a Replication-stream sink; see HBASE-18846 for more. -524 */ -525 private final boolean masterless; -526 static final String MASTERLESS_CONFIG_NAME = "hbase.masterless"; +249 +250 /** +251 * Cluster connection to be shared by services. +252 * Initialized at server startup and closed when server shuts down. +253 * Clients must never close it explicitly. +254 */ +255 protected ClusterConnection clusterConnection; +256 +257 /* +258 * Long-living meta table locator, which is created when the server is started and stopped +259 * when server shuts down. References to this locator shall be used to perform according +260 * operations in EventHandlers. Primary reason for this decision is to make it mockable +261 * for tests. +262 */ +263 protected MetaTableLocator metaTableLocator; +264 +265 /** +266 * Go here to get table descriptors. +267 */ +268 protected TableDescriptors tableDescriptors; +269 +270 // Replication services. If no replication, this handler will be null. +271 protected ReplicationSourceService replicationSourceHandler; +272 protected ReplicationSinkService replicationSinkHandler; +273 +274 // Compactions +275 public CompactSplit compactSplitThread; +276 +277 /** +278 * Map of regions currently being served by this region server. Key is the +279 * encoded region name. All access should be synchronized. +280 */ +281 protected final Map<String, HRegion> onlineRegions = new ConcurrentHashMap<>(); +282 +283 /** +284 * Map of encoded region names to the DataNode locations they should be hosted on +285 * We store the value as InetSocketAddress since this is used only in HDFS +286 * API (create() that takes favored nodes as hints for placing file blocks). +287 * We could have used ServerName here as the value class, but we'd need to +288 * convert it to InetSocketAddress at some point before the HDFS API call, and +289 * it seems a bit weird to store ServerName since ServerName refers to RegionServers +290 * and here we really mean DataNode locations. +291 */ +292 protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap = +293 new ConcurrentHashMap<>(); +294 +295 // Leases +296 protected Leases leases; +297 +298 // Instance of the hbase executor executorService. +299 protected ExecutorService executorService; +300 +301 // If false, the file system has become unavailable +302 protected volatile boolean fsOk; +303 protected HFileSystem fs; +304 protected HFileSystem walFs; +305 +306 // Set when a report to the master comes back with a message asking us to +307 // shutdown. Also set by call to stop when debugging or running unit tests +308 // of HRegionServer in isolation. +309 private volatile boolean stopped = false; +310 +311 // Go down hard. Used if file system becomes unavailable and also in +312 // debugging and unit tests. +313 private volatile boolean abortRequested; +314 +315 ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<>(); +316 +317 // A state before we go into stopped state. At this stage we're closing user +318 // space regions. +319 private boolean stopping = false; +320 +321 volatile boolean killed = false; +322 +323 protected final Configuration conf; +324 +325 private Path rootDir; +326 private Path walRootDir; +327 +328 protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); +329 +330 final int numRetries; +331 protected final int threadWakeFrequency; +332 protected final int msgInterval; +333 +334 protected final int numRegionsToReport; +335 +336 // Stub to do region server status calls against the master. +337 private volatile RegionServerStatusService.BlockingInterface rssStub; +338 private volatile LockService.BlockingInterface lockStub; +339 // RPC client. Used to make the stub above that does region server status checking. +340 RpcClient rpcClient; +341 +342 private RpcRetryingCallerFactory rpcRetryingCallerFactory; +343 private RpcControllerFactory rpcControllerFactory; +344 +345 private UncaughtExceptionHandler uncaughtExceptionHandler; +346 +347 // Info server. Default access so can be used by unit tests. REGIONSERVER +348 // is name of the webapp and the attribute name used stuffing this instance +349 // into web context. +350 protected InfoServer infoServer; +351 private JvmPauseMonitor pauseMonitor; +352 +353 /** region server process name */ +354 public static final String REGIONSERVER = "regionserver"; +355 +356 MetricsRegionServer metricsRegionServer; +357 MetricsTable metricsTable; +358 private SpanReceiverHost spanReceiverHost; +359 +360 /** +361 * ChoreService used to schedule tasks that we want to run periodically +362 */ +363 private ChoreService choreService; +364 +365 /* +366 * Check for compactions requests. +367 */ +368 ScheduledChore compactionChecker; +369 +370 /* +371 * Check for flushes +372 */ +373 ScheduledChore periodicFlusher; +374 +375 protected volatile WALFactory walFactory; +376 +377 // WAL roller. log is protected rather than private to avoid +378 // eclipse warning when accessed by inner classes +379 protected LogRoller walRoller; +380 +381 // A thread which calls reportProcedureDone +382 private RemoteProcedureResultReporter procedureResultReporter; +383 +384 // flag set after we're done setting up server threads +385 final AtomicBoolean online = new AtomicBoolean(false); +386 +387 // zookeeper connection and watcher +388 protected ZKWatcher zooKeeper; +389 +390 // master address tracker +391 private MasterAddressTracker masterAddressTracker; +392 +393 // Cluster Status Tracker +394 protected ClusterStatusTracker clusterStatusTracker; +395 +396 // Log Splitting Worker +397 private SplitLogWorker splitLogWorker; +398 +399 // A sleeper that sleeps for msgInterval. +400 protected final Sleeper sleeper; +401 +402 private final int operationTimeout; +403 private final int shortOperationTimeout; +404 +405 private final RegionServerAccounting regionServerAccounting; +406 +407 // Cache configuration and block cache reference +408 protected CacheConfig cacheConfig; +409 // Cache configuration for mob +410 final MobCacheConfig mobCacheConfig; +411 +412 /** The health check chore. */ +413 private HealthCheckChore healthCheckChore; +414 +415 /** The nonce manager chore. */ +416 private ScheduledChore nonceManagerChore; +417 +418 private Map<String, com.google.protobuf.Service> coprocessorServiceHandlers = Maps.newHashMap(); +419 +420 /** +421 * The server name the Master sees us as. Its made from the hostname the +422 * master passes us, port, and server startcode. Gets set after registration +423 * against Master. +424 */ +425 protected ServerName serverName; +426 +427 /* +428 * hostname specified by hostname config +429 */ +430 protected String useThisHostnameInstead; +431 +432 // key to the config parameter of server hostname +433 // the specification of server hostname is optional. The hostname should be resolvable from +434 // both master and region server +435 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +436 final static String RS_HOSTNAME_KEY = "hbase.regionserver.hostname"; +437 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +438 protected final static String MASTER_HOSTNAME_KEY = "hbase.master.hostname"; +439 +440 // HBASE-18226: This config and hbase.regionserver.hostname are mutually exclusive. +441 // Exception will be thrown if both are used. +442 final static String RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY = +443 "hbase.regionserver.hostname.disable.master.reversedns"; +444 +445 /** +446 * This servers startcode. +447 */ +448 protected final long startcode; +449 +450 /** +451 * Unique identifier for the cluster we are a part of. +452 */ +453 private String clusterId; +454 +455 /** +456 * MX Bean for RegionServerInfo +457 */ +458 private ObjectName mxBean = null; +459 +460 /** +461 * Chore to clean periodically the moved region list +462 */ +463 private MovedRegionsCleaner movedRegionsCleaner; +464 +465 // chore for refreshing store files for secondary regions +466 private StorefileRefresherChore storefileRefresher; +467 +468 private RegionServerCoprocessorHost rsHost; +469 +470 private RegionServerProcedureManagerHost rspmHost; +471 +472 private RegionServerRpcQuotaManager rsQuotaManager; +473 private RegionServerSpaceQuotaManager rsSpaceQuotaManager; +474 +475 /** +476 * Nonce manager. Nonces are used to make operations like increment and append idempotent +477 * in the case where client doesn't receive the response from a successful operation and +478 * retries. We track the successful ops for some time via a nonce sent by client and handle +479 * duplicate operations (currently, by failing them; in future we might use MVCC to return +480 * result). Nonces are also recovered from WAL during, recovery; however, the caveats (from +481 * HBASE-3787) are: +482 * - WAL recovery is optimized, and under high load we won't read nearly nonce-timeout worth +483 * of past records. If we don't read the records, we don't read and recover the nonces. +484 * Some WALs within nonce-timeout at recovery may not even be present due to rolling/cleanup. +485 * - There's no WAL recovery during normal region move, so nonces will not be transfered. +486 * We can have separate additional "Nonce WAL". It will just contain bunch of numbers and +487 * won't be flushed on main path - because WAL itself also contains nonces, if we only flush +488 * it before memstore flush, for a given nonce we will either see it in the WAL (if it was +489 * never flushed to disk, it will be part of recovery), or we'll see it as part of the nonce +490 * log (or both occasionally, which doesn't matter). Nonce log file can be deleted after the +491 * latest nonce in it expired. It can also be recovered during move. +492 */ +493 final ServerNonceManager nonceManager; +494 +495 private UserProvider userProvider; +496 +497 protected final RSRpcServices rpcServices; +498 +499 protected CoordinatedStateManager csm; +500 +501 /** +502 * Configuration manager is used to register/deregister and notify the configuration observers +503 * when the regionserver is notified that there was a change in the on disk configs. +504 */ +505 protected final ConfigurationManager configurationManager; +506 +507 @VisibleForTesting +508 CompactedHFilesDischarger compactedFileDischarger; +509 +510 private volatile ThroughputController flushThroughputController; +511 +512 protected SecureBulkLoadManager secureBulkLoadManager; +513 +514 protected FileSystemUtilizationChore fsUtilizationChore; +515 +516 private final NettyEventLoopGroupConfig eventLoopGroupConfig; +517 +518 /** +519 * True if this RegionServer is coming up in a cluster where there is no Master; +520 * means it needs to just come up and make do without a Master to talk to: e.g. in test or +521 * HRegionServer is doing other than its usual duties: e.g. as an hollowed-out host whose only +522 * purpose is as a Replication-stream sink; see HBASE-18846 for more. +523 */ +524 private final boolean masterless; +525 static final String MASTERLESS_CONFIG_NAME = "hbase.masterless"; +526 527 -528 -529 /** -530 * Starts a HRegionServer at the default location -531 */ -532 // Don't start any services or managers in here in the Constructor. -533 // Defer till after we register with the Master as much as possible. See #startServices. -534 public HRegionServer(Configuration conf) throws IOException { -535 super("RegionServer"); // thread name -536 TraceUtil.initTracer(conf); -537 try { -538 this.startcode = System.currentTimeMillis(); -539 this.conf = conf; -540 this.fsOk = true; -541 this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); -542 this.eventLoopGroupConfig = setupNetty(this.conf); -543 MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); -544 HFile.checkHFileVersion(this.conf); -545 checkCodecs(this.conf); -546 this.userProvider = UserProvider.instantiate(conf); -547 FSUtils.setupShortCircuitRead(this.conf); -548 Replication.decorateRegionServerConfiguration(this.conf); -549 -550 // Disable usage of meta replicas in the regionserver -551 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); -552 // Config'ed params -553 this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, -554 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); -555 this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); -556 this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000); -557 -558 this.sleeper = new Sleeper(this.msgInterval, this); -559 -560 boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true); -561 this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null; -562 -563 this.numRegionsToReport = conf.getInt("hbase.regionserver.numregionstoreport", 10); -564 -565 this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, -566 HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); -567 -568 this.shortOperationTimeout = conf.getInt(HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY, -569 HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT); -570 -571 this.abortRequested = false; -572 this.stopped = false; -573 -574 rpcServices = createRpcServices(); -575 if (this instanceof HMaster) { -576 useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY); -577 } else { -578 useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY); -579 if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) { -580 if (shouldUseThisHostnameInstead()) { -581 String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY + -582 " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + -583 " to true while " + RS_HOSTNAME_KEY + " is used"; -584 throw new IOException(msg); -585 } else { -586 useThisHostnameInstead = rpcServices.isa.getHostName(); -587 } -588 } -589 } -590 String hostName = shouldUseThisHostnameInstead() ? -591 this.useThisHostnameInstead : this.rpcServices.isa.getHostName(); -592 serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); -593 -594 rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); -595 rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf); -596 -597 // login the zookeeper client principal (if using security) -598 ZKUtil.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, -599 HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName); -600 // login the server principal (if using secure Hadoop) -601 login(userProvider, hostName); -602 // init superusers and add the server principal (if using security) -603 // or process owner as default super user. -604 Superusers.initialize(conf); -605 -606 regionServerAccounting = new RegionServerAccounting(conf); -607 cacheConfig = new CacheConfig(conf); -608 mobCacheConfig = new MobCacheConfig(conf); -609 uncaughtExceptionHandler = new UncaughtExceptionHandler() { -610 @Override -611 public void uncaughtException(Thread t, Throwable e) { -612 abort("Uncaught exception in executorService thread " + t.getName(), e); -613 } -614 }; -615 -616 initializeFileSystem(); -617 spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); -618 -619 this.configurationManager = new ConfigurationManager(); -620 setupWindows(getConfiguration(), getConfigurationManager()); -621 -622 // Some unit tests don't need a cluster, so no zookeeper at all -623 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -624 // Open connection to zookeeper and set primary watcher -625 zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + -626 rpcServices.isa.getPort(), this, canCreateBaseZNode()); -627 -628 // If no master in cluster, skip trying to track one or look for a cluster status. -629 if (!this.masterless) { -630 this.csm = new ZkCoordinatedStateManager(this); -631 -632 masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this); -633 masterAddressTracker.start(); -634 -635 clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this); -636 clusterStatusTracker.start(); -637 } -638 } -639 // This violates 'no starting stuff in Constructor' but Master depends on the below chore -640 // and executor being created and takes a different startup route. Lots of overlap between HRS -641 // and M (An M IS A HRS now). Need to refactor so less duplication between M and its super -642 // Master expects Constructor to put up web servers. Ugh. -643 // class HRS. TODO. -644 this.choreService = new ChoreService(getServerName().toString(), true); -645 this.executorService = new ExecutorService(getServerName().toShortString()); -646 this.rpcServices.start(); -647 putUpWebUI(); -648 } catch (Throwable t) { -649 // Make sure we log the exception. HRegionServer is often started via reflection and the -650 // cause of failed startup is lost. -651 LOG.error("Failed construction RegionServer", t); -652 throw t; -653 } -654 } -655 -656 /** -657 * If running on Windows, do windows-specific setup. -658 */ -659 private static void setupWindows(final Configuration conf, ConfigurationManager cm) { -660 if (!SystemUtils.IS_OS_WINDOWS) { -661 Signal.handle(new Signal("HUP"), new SignalHandler() { -662 @Override -663 public void handle(Signal signal) { -664 conf.reloadConfiguration(); -665 cm.notifyAllObservers(conf); -666 } -667 }); -668 } -669 } -670 -671 private static NettyEventLoopGroupConfig setupNetty(Configuration conf) { -672 // Initialize netty event loop group at start as we may use it for rpc server, rpc client & WAL. -673 NettyEventLoopGroupConfig nelgc = -674 new NettyEventLoopGroupConfig(conf, "RS-EventLoopGroup"); -675 NettyRpcClientConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); -676 NettyAsyncFSWALConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); -677 return nelgc; -678 } -679 -680 private void initializeFileSystem() throws IOException { -681 // Get fs instance used by this RS. Do we use checksum verification in the hbase? If hbase -682 // checksum verification enabled, then automatically switch off hdfs checksum verification. -683 boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true); -684 FSUtils.setFsDefault(this.conf, FSUtils.getWALRootDir(this.conf)); -685 this.walFs = new HFileSystem(this.conf, useHBaseChecksum); -686 this.walRootDir = FSUtils.getWALRootDir(this.conf); -687 // Set 'fs.defaultFS' to match the filesystem on hbase.rootdir else -688 // underlying hadoop hdfs accessors will be going against wrong filesystem -689 // (unless all is set to defaults). -690 FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf)); -691 this.fs = new HFileSystem(this.conf, useHBaseChecksum); -692 this.rootDir = FSUtils.getRootDir(this.conf); -693 this.tableDescriptors = getFsTableDescriptors(); -694 } -695 -696 protected TableDescriptors getFsTableDescriptors() throws IOException { -697 return new FSTableDescriptors(this.conf, -698 this.fs, this.rootDir, !canUpdateTableDescriptor(), false, getMetaTableObserver()); -699 } -700 -701 protected Function<TableDescriptorBuilder, TableDescriptorBuilder> getMetaTableObserver() { -702 return null; -703 } -704 -705 protected void setInitLatch(CountDownLatch latch) { -706 this.initLatch = latch; -707 } -708 -709 /* -710 * Returns true if configured hostname should be used -711 */ -712 protected boolean shouldUseThisHostnameInstead() { -713 return useThisHostnameInstead != null && !useThisHostnameInstead.isEmpty(); +528 /** +529 * Starts a HRegionServer at the default location +530 */ +531 // Don't start any services or managers in here in the Constructor. +532 // Defer till after we register with the Master as much as possible. See #startServices. +533 public HRegionServer(Configuration conf) throws IOException { +534 super("RegionServer"); // thread name +535 TraceUtil.initTracer(conf); +536 try { +537 this.startcode = System.currentTimeMillis(); +538 this.conf = conf; +539 this.fsOk = true; +540 this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); +541 this.eventLoopGroupConfig = setupNetty(this.conf); +542 MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); +543 HFile.checkHFileVersion(this.conf); +544 checkCodecs(this.conf); +545 this.userProvider = UserProvider.instantiate(conf); +546 FSUtils.setupShortCircuitRead(this.conf); +547 Replication.decorateRegionServerConfiguration(this.conf); +548 +549 // Disable usage of meta replicas in the regionserver +550 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); +551 // Config'ed params +552 this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, +553 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); +554 this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); +555 this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000); +556 +557 this.sleeper = new Sleeper(this.msgInterval, this); +558 +559 boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true); +560 this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null; +561 +562 this.numRegionsToReport = conf.getInt("hbase.regionserver.numregionstoreport", 10); +563 +564 this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, +565 HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); +566 +567 this.shortOperationTimeout = conf.getInt(HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY, +568 HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT); +569 +570 this.abortRequested = false; +571 this.stopped = false; +572 +573 rpcServices = createRpcServices(); +574 if (this instanceof HMaster) { +575 useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY); +576 } else { +577 useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY); +578 if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) { +579 if (shouldUseThisHostnameInstead()) { +580 String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY + +581 " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + +582 " to true while " + RS_HOSTNAME_KEY + " is used"; +583 throw new IOException(msg); +584 } else { +585 useThisHostnameInstead = rpcServices.isa.getHostName(); +586 } +587 } +588 } +589 String hostName = shouldUseThisHostnameInstead() ? +590 this.useThisHostnameInstead : this.rpcServices.isa.getHostName(); +591 serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); +592 +593 rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); +594 rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf); +595 +596 // login the zookeeper client principal (if using security) +597 ZKUtil.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, +598 HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName); +599 // login the server principal (if using secure Hadoop) +600 login(userProvider, hostName); +601 // init superusers and add the server principal (if using security) +602 // or process owner as default super user. +603 Superusers.initialize(conf); +604 +605 regionServerAccounting = new RegionServerAccounting(conf); +606 cacheConfig = new CacheConfig(conf); +607 mobCacheConfig = new MobCacheConfig(conf); +608 uncaughtExceptionHandler = new UncaughtExceptionHandler() { +609 @Override +610 public void uncaughtException(Thread t, Throwable e) { +611 abort("Uncaught exception in executorService thread " + t.getName(), e); +612 } +613 }; +614 +615 initializeFileSystem(); +616 spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); +617 +618 this.configurationManager = new ConfigurationManager(); +619 setupWindows(getConfiguration(), getConfigurationManager()); +620 +621 // Some unit tests don't need a cluster, so no zookeeper at all +622 if (!conf.getBoolean("hbase.testing.nocluster", false)) { +623 // Open connection to zookeeper and set primary watcher +624 zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + +625 rpcServices.isa.getPort(), this, canCreateBaseZNode()); +626 +627 // If no master in cluster, skip trying to track one or look for a cluster status. +628 if (!this.masterless) { +629 this.csm = new ZkCoordinatedStateManager(this); +630 +631 masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this); +632 masterAddressTracker.start(); +633 +634 clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this); +635 clusterStatusTracker.start(); +636 } +637 } +638 // This violates 'no starting stuff in Constructor' but Master depends on the below chore +639 // and executor being created and takes a different startup route. Lots of overlap between HRS +640 // and M (An M IS A HRS now). Need to refactor so less duplication between M and its super +641 // Master expects Constructor to put up web servers. Ugh. +642 // class HRS. TODO. +643 this.choreService = new ChoreService(getServerName().toString(), true); +644 this.executorService = new ExecutorService(getServerName().toShortString()); +645 this.rpcServices.start(); +646 putUpWebUI(); +647 } catch (Throwable t) { +648 // Make sure we log the exception. HRegionServer is often started via reflection and the +649 // cause of failed startup is lost. +650 LOG.error("Failed construction RegionServer", t); +651 throw t; +652 } +653 } +654 +655 /** +656 * If running on Windows, do windows-specific setup. +657 */ +658 private static void setupWindows(final Configuration conf, ConfigurationManager cm) { +659 if (!SystemUtils.IS_OS_WINDOWS) { +660 Signal.handle(new Signal("HUP"), new SignalHandler() { +661 @Override +662 public void handle(Signal signal) { +