From commits-return-65546-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Mon Jan 15 16:31:32 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id E4F15180792 for ; Mon, 15 Jan 2018 16:31:31 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D56F0160C48; Mon, 15 Jan 2018 15:31:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1080A160C4F for ; Mon, 15 Jan 2018 16:31:27 +0100 (CET) Received: (qmail 73091 invoked by uid 500); 15 Jan 2018 15:31:25 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 72837 invoked by uid 99); 15 Jan 2018 15:31:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 15 Jan 2018 15:31:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 59869F17C8; Mon, 15 Jan 2018 15:31:24 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Mon, 15 Jan 2018 15:31:37 -0000 Message-Id: <998396377c7c4a7b8549913720e4b22d@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/30] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e80e3339/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.MovedRegionsCleaner.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.MovedRegionsCleaner.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.MovedRegionsCleaner.html index 2bbae5e..9ddbcd4 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.MovedRegionsCleaner.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegionServer.MovedRegionsCleaner.html @@ -33,32 +33,32 @@ 025import java.net.BindException; 026import java.net.InetAddress; 027import java.net.InetSocketAddress; -028import java.util.ArrayList; -029import java.util.Collection; -030import java.util.Collections; -031import java.util.Comparator; -032import java.util.HashSet; -033import java.util.Iterator; -034import java.util.List; -035import java.util.Map; -036import java.util.Map.Entry; -037import java.util.Objects; -038import java.util.Set; -039import java.util.SortedMap; -040import java.util.TreeMap; -041import java.util.TreeSet; -042import java.util.concurrent.ConcurrentHashMap; -043import java.util.concurrent.ConcurrentMap; -044import java.util.concurrent.ConcurrentSkipListMap; -045import java.util.concurrent.CountDownLatch; -046import java.util.concurrent.TimeUnit; -047import java.util.concurrent.atomic.AtomicBoolean; -048import java.util.concurrent.locks.ReentrantReadWriteLock; -049import java.util.function.Function; -050import javax.management.MalformedObjectNameException; -051import javax.management.ObjectName; -052import javax.servlet.http.HttpServlet; -053import org.apache.commons.lang3.RandomUtils; +028import java.time.Duration; +029import java.util.ArrayList; +030import java.util.Collection; +031import java.util.Collections; +032import java.util.Comparator; +033import java.util.HashSet; +034import java.util.Iterator; +035import java.util.List; +036import java.util.Map; +037import java.util.Map.Entry; +038import java.util.Objects; +039import java.util.Set; +040import java.util.SortedMap; +041import java.util.TreeMap; +042import java.util.TreeSet; +043import java.util.concurrent.ConcurrentHashMap; +044import java.util.concurrent.ConcurrentMap; +045import java.util.concurrent.ConcurrentSkipListMap; +046import java.util.concurrent.atomic.AtomicBoolean; +047import java.util.concurrent.locks.ReentrantReadWriteLock; +048import java.util.function.Function; +049import javax.management.MalformedObjectNameException; +050import javax.management.ObjectName; +051import javax.servlet.http.HttpServlet; +052import org.apache.commons.lang3.RandomUtils; +053import org.apache.commons.lang3.StringUtils; 054import org.apache.commons.lang3.SystemUtils; 055import org.apache.hadoop.conf.Configuration; 056import org.apache.hadoop.fs.FileSystem; @@ -177,16 +177,16 @@ 169import org.apache.hadoop.ipc.RemoteException; 170import org.apache.hadoop.metrics2.util.MBeans; 171import org.apache.hadoop.util.ReflectionUtils; -172import org.apache.hadoop.util.StringUtils; -173import org.apache.yetus.audience.InterfaceAudience; -174import org.apache.zookeeper.KeeperException; -175import org.slf4j.Logger; -176import org.slf4j.LoggerFactory; -177import sun.misc.Signal; -178import sun.misc.SignalHandler; -179 -180import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -181import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +172import org.apache.yetus.audience.InterfaceAudience; +173import org.apache.zookeeper.KeeperException; +174import org.slf4j.Logger; +175import org.slf4j.LoggerFactory; +176import sun.misc.Signal; +177import sun.misc.SignalHandler; +178 +179import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +180import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +181import org.apache.hbase.thirdparty.com.google.common.base.Throwables; 182import org.apache.hbase.thirdparty.com.google.common.collect.Maps; 183import org.apache.hbase.thirdparty.com.google.protobuf.BlockingRpcChannel; 184import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; @@ -393,13 +393,13 @@ 385 final AtomicBoolean online = new AtomicBoolean(false); 386 387 // zookeeper connection and watcher -388 protected ZKWatcher zooKeeper; +388 protected final ZKWatcher zooKeeper; 389 390 // master address tracker -391 private MasterAddressTracker masterAddressTracker; +391 private final MasterAddressTracker masterAddressTracker; 392 393 // Cluster Status Tracker -394 protected ClusterStatusTracker clusterStatusTracker; +394 protected final ClusterStatusTracker clusterStatusTracker; 395 396 // Log Splitting Worker 397 private SplitLogWorker splitLogWorker; @@ -532,3227 +532,3219 @@ 524 private final boolean masterless; 525 static final String MASTERLESS_CONFIG_NAME = "hbase.masterless"; 526 -527 -528 /** -529 * Starts a HRegionServer at the default location -530 */ -531 // Don't start any services or managers in here in the Constructor. -532 // Defer till after we register with the Master as much as possible. See #startServices. -533 public HRegionServer(Configuration conf) throws IOException { -534 super("RegionServer"); // thread name -535 TraceUtil.initTracer(conf); -536 try { -537 this.startcode = System.currentTimeMillis(); -538 this.conf = conf; -539 this.fsOk = true; -540 this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); -541 this.eventLoopGroupConfig = setupNetty(this.conf); -542 MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); -543 HFile.checkHFileVersion(this.conf); -544 checkCodecs(this.conf); -545 this.userProvider = UserProvider.instantiate(conf); -546 FSUtils.setupShortCircuitRead(this.conf); -547 Replication.decorateRegionServerConfiguration(this.conf); -548 -549 // Disable usage of meta replicas in the regionserver -550 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); -551 // Config'ed params -552 this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, -553 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); -554 this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); -555 this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000); -556 -557 this.sleeper = new Sleeper(this.msgInterval, this); -558 -559 boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true); -560 this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null; -561 -562 this.numRegionsToReport = conf.getInt("hbase.regionserver.numregionstoreport", 10); -563 -564 this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, -565 HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); -566 -567 this.shortOperationTimeout = conf.getInt(HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY, -568 HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT); -569 -570 this.abortRequested = false; -571 this.stopped = false; -572 -573 rpcServices = createRpcServices(); -574 if (this instanceof HMaster) { -575 useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY); -576 } else { -577 useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY); -578 if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) { -579 if (shouldUseThisHostnameInstead()) { -580 String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY + -581 " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + -582 " to true while " + RS_HOSTNAME_KEY + " is used"; -583 throw new IOException(msg); -584 } else { -585 useThisHostnameInstead = rpcServices.isa.getHostName(); -586 } -587 } -588 } -589 String hostName = shouldUseThisHostnameInstead() ? -590 this.useThisHostnameInstead : this.rpcServices.isa.getHostName(); -591 serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); -592 -593 rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); -594 rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf); -595 -596 // login the zookeeper client principal (if using security) -597 ZKUtil.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, -598 HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName); -599 // login the server principal (if using secure Hadoop) -600 login(userProvider, hostName); -601 // init superusers and add the server principal (if using security) -602 // or process owner as default super user. -603 Superusers.initialize(conf); -604 -605 regionServerAccounting = new RegionServerAccounting(conf); -606 cacheConfig = new CacheConfig(conf); -607 mobCacheConfig = new MobCacheConfig(conf); -608 uncaughtExceptionHandler = new UncaughtExceptionHandler() { -609 @Override -610 public void uncaughtException(Thread t, Throwable e) { -611 abort("Uncaught exception in executorService thread " + t.getName(), e); -612 } -613 }; -614 -615 initializeFileSystem(); -616 spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); -617 -618 this.configurationManager = new ConfigurationManager(); -619 setupWindows(getConfiguration(), getConfigurationManager()); -620 -621 // Some unit tests don't need a cluster, so no zookeeper at all -622 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -623 // Open connection to zookeeper and set primary watcher -624 zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + -625 rpcServices.isa.getPort(), this, canCreateBaseZNode()); -626 -627 // If no master in cluster, skip trying to track one or look for a cluster status. -628 if (!this.masterless) { -629 this.csm = new ZkCoordinatedStateManager(this); -630 -631 masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this); -632 masterAddressTracker.start(); -633 -634 clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this); -635 clusterStatusTracker.start(); -636 } -637 } -638 // This violates 'no starting stuff in Constructor' but Master depends on the below chore -639 // and executor being created and takes a different startup route. Lots of overlap between HRS -640 // and M (An M IS A HRS now). Need to refactor so less duplication between M and its super -641 // Master expects Constructor to put up web servers. Ugh. -642 // class HRS. TODO. -643 this.choreService = new ChoreService(getServerName().toString(), true); -644 this.executorService = new ExecutorService(getServerName().toShortString()); -645 this.rpcServices.start(); -646 putUpWebUI(); -647 } catch (Throwable t) { -648 // Make sure we log the exception. HRegionServer is often started via reflection and the -649 // cause of failed startup is lost. -650 LOG.error("Failed construction RegionServer", t); -651 throw t; -652 } -653 } -654 -655 /** -656 * If running on Windows, do windows-specific setup. -657 */ -658 private static void setupWindows(final Configuration conf, ConfigurationManager cm) { -659 if (!SystemUtils.IS_OS_WINDOWS) { -660 Signal.handle(new Signal("HUP"), new SignalHandler() { -661 @Override -662 public void handle(Signal signal) { -663 conf.reloadConfiguration(); -664 cm.notifyAllObservers(conf); -665 } -666 }); -667 } -668 } -669 -670 private static NettyEventLoopGroupConfig setupNetty(Configuration conf) { -671 // Initialize netty event loop group at start as we may use it for rpc server, rpc client & WAL. -672 NettyEventLoopGroupConfig nelgc = -673 new NettyEventLoopGroupConfig(conf, "RS-EventLoopGroup"); -674 NettyRpcClientConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); -675 NettyAsyncFSWALConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); -676 return nelgc; +527 /** +528 * Starts a HRegionServer at the default location +529 */ +530 // Don't start any services or managers in here in the Constructor. +531 // Defer till after we register with the Master as much as possible. See #startServices. +532 public HRegionServer(Configuration conf) throws IOException { +533 super("RegionServer"); // thread name +534 TraceUtil.initTracer(conf); +535 try { +536 this.startcode = System.currentTimeMillis(); +537 this.conf = conf; +538 this.fsOk = true; +539 this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); +540 this.eventLoopGroupConfig = setupNetty(this.conf); +541 MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); +542 HFile.checkHFileVersion(this.conf); +543 checkCodecs(this.conf); +544 this.userProvider = UserProvider.instantiate(conf); +545 FSUtils.setupShortCircuitRead(this.conf); +546 Replication.decorateRegionServerConfiguration(this.conf); +547 +548 // Disable usage of meta replicas in the regionserver +549 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); +550 // Config'ed params +551 this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, +552 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); +553 this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); +554 this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000); +555 +556 this.sleeper = new Sleeper(this.msgInterval, this); +557 +558 boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true); +559 this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null; +560 +561 this.numRegionsToReport = conf.getInt("hbase.regionserver.numregionstoreport", 10); +562 +563 this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, +564 HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); +565 +566 this.shortOperationTimeout = conf.getInt(HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY, +567 HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT); +568 +569 this.abortRequested = false; +570 this.stopped = false; +571 +572 rpcServices = createRpcServices(); +573 useThisHostnameInstead = getUseThisHostnameInstead(conf); +574 String hostName = +575 StringUtils.isBlank(useThisHostnameInstead) ? this.rpcServices.isa.getHostName() +576 : this.useThisHostnameInstead; +577 serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); +578 +579 rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); +580 rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf); +581 +582 // login the zookeeper client principal (if using security) +583 ZKUtil.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, +584 HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName); +585 // login the server principal (if using secure Hadoop) +586 login(userProvider, hostName); +587 // init superusers and add the server principal (if using security) +588 // or process owner as default super user. +589 Superusers.initialize(conf); +590 +591 regionServerAccounting = new RegionServerAccounting(conf); +592 cacheConfig = new CacheConfig(conf); +593 mobCacheConfig = new MobCacheConfig(conf); +594 uncaughtExceptionHandler = new UncaughtExceptionHandler() { +595 @Override +596 public void uncaughtException(Thread t, Throwable e) { +597 abort("Uncaught exception in executorService thread " + t.getName(), e); +598 } +599 }; +600 +601 initializeFileSystem(); +602 spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); +603 +604 this.configurationManager = new ConfigurationManager(); +605 setupWindows(getConfiguration(), getConfigurationManager()); +606 +607 // Some unit tests don't need a cluster, so no zookeeper at all +608 if (!conf.getBoolean("hbase.testing.nocluster", false)) { +609 // Open connection to zookeeper and set primary watcher +610 zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + +611 rpcServices.isa.getPort(), this, canCreateBaseZNode()); +612 // If no master in cluster, skip trying to track one or look for a cluster status. +613 if (!this.masterless) { +614 this.csm = new ZkCoordinatedStateManager(this); +615 +616 masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this); +617 masterAddressTracker.start(); +618 +619 clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this); +620 clusterStatusTracker.start(); +621 } else { +622 masterAddressTracker = null; +623 clusterStatusTracker = null; +624 } +625 } else { +626 zooKeeper = null; +627 masterAddressTracker = null; +628 clusterStatusTracker = null; +629 } +630 // This violates 'no starting stuff in Constructor' but Master depends on the below chore +631 // and executor being created and takes a different startup route. Lots of overlap between HRS +632 // and M (An M IS A HRS now). Need to refactor so less duplication between M and its super +633 // Master expects Constructor to put up web servers. Ugh. +634 // class HRS. TODO. +635 this.choreService = new ChoreService(getServerName().toString(), true); +636 this.executorService = new ExecutorService(getServerName().toShortString()); +637 this.rpcServices.start(); +638 putUpWebUI(); +639 } catch (Throwable t) { +640 // Make sure we log the exception. HRegionServer is often started via reflection and the +641 // cause of failed startup is lost. +642 LOG.error("Failed construction RegionServer", t); +643 throw t; +644 } +645 } +646 +647 // HMaster should override this method to load the specific config for master +648 protected String getUseThisHostnameInstead(Configuration conf) throws IOException { +649 String hostname = conf.get(RS_HOSTNAME_KEY); +650 if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) { +651 if (!StringUtils.isBlank(hostname)) { +652 String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY + +653 " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + +654 " to true while " + RS_HOSTNAME_KEY + " is used"; +655 throw new IOException(msg); +656 } else { +657 return rpcServices.isa.getHostName(); +658 } +659 } else { +660 return hostname; +661 } +662 } +663 +664 /** +665 * If running on Windows, do windows-specific setup. +666 */ +667 private static void setupWindows(final Configuration conf, ConfigurationManager cm) { +668 if (!SystemUtils.IS_OS_WINDOWS) { +669 Signal.handle(new Signal("HUP"), new SignalHandler() { +670 @Override +671 public void handle(Signal signal) { +672 conf.reloadConfiguration(); +673 cm.notifyAllObservers(conf); +674 } +675 }); +676 } 677 } 678 -679 private void initializeFileSystem() throws IOException { -680 // Get fs instance used by this RS. Do we use checksum verification in the hbase? If hbase -681 // checksum verification enabled, then automatically switch off hdfs checksum verification. -682 boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true); -683 FSUtils.setFsDefault(this.conf, FSUtils.getWALRootDir(this.conf)); -684 this.walFs = new HFileSystem(this.conf, useHBaseChecksum); -685 this.walRootDir = FSUtils.getWALRootDir(this.conf); -686 // Set 'fs.defaultFS' to match the filesystem on hbase.rootdir else -687 // underlying hadoop hdfs accessors will be going against wrong filesystem -688 // (unless all is set to defaults). -689 FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf)); -690 this.fs = new HFileSystem(this.conf, useHBaseChecksum); -691 this.rootDir = FSUtils.getRootDir(this.conf); -692 this.tableDescriptors = getFsTableDescriptors(); -693 } -694 -695 protected TableDescriptors getFsTableDescriptors() throws IOException { -696 return new FSTableDescriptors(this.conf, -697 this.fs, this.rootDir, !canUpdateTableDescriptor(), false, getMetaTableObserver()); -698 } -699 -700 protected Function<TableDescriptorBuilder, TableDescriptorBuilder> getMetaTableObserver() { -701 return null; +679 private static NettyEventLoopGroupConfig setupNetty(Configuration conf) { +680 // Initialize netty event loop group at start as we may use it for rpc server, rpc client & WAL. +681 NettyEventLoopGroupConfig nelgc = +682 new NettyEventLoopGroupConfig(conf, "RS-EventLoopGroup"); +683 NettyRpcClientConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); +684 NettyAsyncFSWALConfigHelper.setEventLoopConfig(conf, nelgc.group(), nelgc.clientChannelClass()); +685 return nelgc; +686 } +687 +688 private void initializeFileSystem() throws IOException { +689 // Get fs instance used by this RS. Do we use checksum verification in the hbase? If hbase +690 // checksum verification enabled, then automatically switch off hdfs checksum verification. +691 boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true); +692 FSUtils.setFsDefault(this.conf, FSUtils.getWALRootDir(this.conf)); +693 this.walFs = new HFileSystem(this.conf, useHBaseChecksum); +694 this.walRootDir = FSUtils.getWALRootDir(this.conf); +695 // Set 'fs.defaultFS' to match the filesystem on hbase.rootdir else +696 // underlying hadoop hdfs accessors will be going against wrong filesystem +697 // (unless all is set to defaults). +698 FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf)); +699 this.fs = new HFileSystem(this.conf, useHBaseChecksum); +700 this.rootDir = FSUtils.getRootDir(this.conf); +701 this.tableDescriptors = getFsTableDescriptors(); 702 } 703 -704 /* -705 * Returns true if configured hostname should be used -706 */ -707 protected boolean shouldUseThisHostnameInstead() { -708 return useThisHostnameInstead != null && !useThisHostnameInstead.isEmpty(); -709 } -710 -711 protected void login(UserProvider user, String host) throws IOException { -712 user.login("hbase.regionserver.keytab.file", -713 "hbase.regionserver.kerberos.principal", host); -714 } -715 -716 protected void waitForMasterActive() { -717 } -718 -719 protected String getProcessName() { -720 return REGIONSERVER; -721 } -722 -723 protected boolean canCreateBaseZNode() { -724 return this.masterless; -725 } -726 -727 protected boolean canUpdateTableDescriptor() { -728 return false; -729 } -730 -731 protected RSRpcServices createRpcServices() throws IOException { -732 return new RSRpcServices(this); -733 } -734 -735 protected void configureInfoServer() { -736 infoServer.addServlet("rs-status", "/rs-status", RSStatusServlet.class); -737 infoServer.setAttribute(REGIONSERVER, this); -738 } -739 -740 protected Class<? extends HttpServlet> getDumpServlet() { -741 return RSDumpServlet.class; -742 } -743 -744 @Override -745 public boolean registerService(com.google.protobuf.Service instance) { -746 /* -747 * No stacking of instances is allowed for a single executorService name -748 */ -749 com.google.protobuf.Descriptors.ServiceDescriptor serviceDesc = -750 instance.getDescriptorForType(); -751 String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc); -752 if (coprocessorServiceHandlers.containsKey(serviceName)) { -753 LOG.error("Coprocessor executorService " + serviceName -754 + " already registered, rejecting request from " + instance); -755 return false; -756 } -757 -758 coprocessorServiceHandlers.put(serviceName, instance); -759 if (LOG.isDebugEnabled()) { -760 LOG.debug("Registered regionserver coprocessor executorService: executorService=" + serviceName); -761 } -762 return true; -763 } -764 -765 /** -766 * Create a 'smarter' Connection, one that is capable of by-passing RPC if the request is to -767 * the local server; i.e. a short-circuit Connection. Safe to use going to local or remote -768 * server. Create this instance in a method can be intercepted and mocked in tests. -769 * @throws IOException -770 */ -771 @VisibleForTesting -772 protected ClusterConnection createClusterConnection() throws IOException { -773 // Create a cluster connection that when appropriate, can short-circuit and go directly to the -774 // local server if the request is to the local server bypassing RPC. Can be used for both local -775 // and remote invocations. -776 return ConnectionUtils.createShortCircuitConnection(conf, null, userProvider.getCurrent(), -777 serverName, rpcServices, rpcServices); -778 } -779 -780 /** -781 * Run test on configured codecs to make sure supporting libs are in place. -782 * @param c -783 * @throws IOException -784 */ -785 private static void checkCodecs(final Configuration c) throws IOException { -786 // check to see if the codec list is available: -787 String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null); -788 if (codecs == null) return; -789 for (String codec : codecs) { -790 if (!CompressionTest.testCompression(codec)) { -791 throw new IOException("Compression codec " + codec + -792 " not supported, aborting RS construction"); -793 } -794 } -795 } -796 -797 public String getClusterId() { -798 return this.clusterId; -799 } -800 -801 /** -802 * Setup our cluster connection if not already initialized. -803 * @throws IOException -804 */ -805 protected synchronized void setupClusterConnection() throws IOException { -806 if (clusterConnection == null) { -807 clusterConnection = createClusterConnection(); -808 metaTableLocator = new MetaTableLocator(); -809 } -810 } -811 -812 /** -813 * All initialization needed before we go register with Master. -814 * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master. -815 * In here we just put up the RpcServer, setup Connection, and ZooKeeper. -816 * -817 * @throws IOException -818 * @throws InterruptedException -819 */ -820 private void preRegistrationInitialization() { -821 try { +704 protected TableDescriptors getFsTableDescriptors() throws IOException { +705 return new FSTableDescriptors(this.conf, +706 this.fs, this.rootDir, !canUpdateTableDescriptor(), false, getMetaTableObserver()); +707 } +708 +709 protected Function<TableDescriptorBuilder, TableDescriptorBuilder> getMetaTableObserver() { +710 return null; +711 } +712 +713 protected void login(UserProvider user, String host) throws IOException { +714 user.login("hbase.regionserver.keytab.file", +715 "hbase.regionserver.kerberos.principal", host); +716 } +717 +718 protected void waitForMasterActive() { +719 } +720 +721 protected String getProcessName() { +722 return REGIONSERVER; +723 } +724 +725 protected boolean canCreateBaseZNode() { +726 return this.masterless; +727 } +728 +729 protected boolean canUpdateTableDescriptor() { +730 return false; +731 } +732 +733 protected RSRpcServices createRpcServices() throws IOException { +734 return new RSRpcServices(this); +735 } +736 +737 protected void configureInfoServer() { +738 infoServer.addServlet("rs-status", "/rs-status", RSStatusServlet.class); +739 infoServer.setAttribute(REGIONSERVER, this); +740 } +741 +742 protected Class<? extends HttpServlet> getDumpServlet() { +743 return RSDumpServlet.class; +744 } +745 +746 @Override +747 public boolean registerService(com.google.protobuf.Service instance) { +748 /* +749 * No stacking of instances is allowed for a single executorService name +750 */ +751 com.google.protobuf.Descriptors.ServiceDescriptor serviceDesc = +752 instance.getDescriptorForType(); +753 String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc); +754 if (coprocessorServiceHandlers.containsKey(serviceName)) { +755 LOG.error("Coprocessor executorService " + serviceName +756 + " already registered, rejecting request from " + instance); +757 return false; +758 } +759 +760 coprocessorServiceHandlers.put(serviceName, instance); +761 if (LOG.isDebugEnabled()) { +762 LOG.debug("Registered regionserver coprocessor executorService: executorService=" + serviceName); +763 } +764 return true; +765 } +766 +767 /** +768 * Create a 'smarter' Connection, one that is capable of by-passing RPC if the request is to +769 * the local server; i.e. a short-circuit Connection. Safe to use going to local or remote +770 * server. Create this instance in a method can be intercepted and mocked in tests. +771 * @throws IOException +772 */ +773 @VisibleForTesting +774 protected ClusterConnection createClusterConnection() throws IOException { +775 // Create a cluster connection that when appropriate, can short-circuit and go directly to the +776 // local server if the request is to the local server bypassing RPC. Can be used for both local +777 // and remote invocations. +778 return ConnectionUtils.createShortCircuitConnection(conf, null, userProvider.getCurrent(), +779 serverName, rpcServices, rpcServices); +780 } +781 +782 /** +783 * Run test on configured codecs to make sure supporting libs are in place. +784 * @param c +785 * @throws IOException +786 */ +787 private static void checkCodecs(final Configuration c) throws IOException { +788 // check to see if the codec list is available: +789 String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null); +790 if (codecs == null) return; +791 for (String codec : codecs) { +792 if (!CompressionTest.testCompression(codec)) { +793 throw new IOException("Compression codec " + codec + +794 " not supported, aborting RS construction"); +795 } +796 } +797 } +798 +799 public String getClusterId() { +800 return this.clusterId; +801 } +802 +803 /** +804 * Setup our cluster connection if not already initialized. +805 * @throws IOException +806 */ +807 protected synchronized void setupClusterConnection() throws IOException { +808 if (clusterConnection == null) { +809 clusterConnection = createClusterConnection(); +810 metaTableLocator = new MetaTableLocator(); +811 } +812 } +813 +814 /** +815 * All initialization needed before we go register with Master.<br> +816 * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master.<br> +817 * In here we just put up the RpcServer, setup Connection, and ZooKeeper. +818 */ +819 private void preRegistrationInitialization() { +820 try { +821 initializeZooKeeper(); 822 setupClusterConnection(); -823 initializeZooKeeper(); -824 // Setup RPC client for master communication -825 this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress( -826 this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics()); -827 } catch (Throwable t) { -828 // Call stop if error or process will stick around for ever since server -829 // puts up non-daemon threads. -830 this.rpcServices.stop(); -831 abort("Initialization of RS failed. Hence aborting RS.", t); -832 } -833 } -834 -835 /** -836 * Bring up connection to zk ensemble and then wait until a master for this -837 * cluster and then after that, wait until cluster 'up' flag has been set. -838 * This is the order in which master does things. -839 * Finally open long-living server short-circuit connection. -840 * @throws IOException -841 * @throws InterruptedException -842 */ -843 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", -844 justification="cluster Id znode read would give us correct response") -845 private void initializeZooKeeper() throws IOException, InterruptedException { -846 // Nothing to do in here if no Master in the mix. -847 if (this.masterless) return; -848 -849 // Create the master address tracker, register with zk, and start it. Then -850 // block until a master is available. No point in starting up if no master -851 // running. -852 blockAndCheckIfStopped(this.masterAddressTracker); -853 -854 // Wait on cluster being up. Master will set this flag up in zookeeper -855 // when ready. -856 blockAndCheckIfStopped(this.clusterStatusTracker); -857 -858 // Retrieve clusterId -859 // Since cluster status is now up -860 // ID should have already been set by HMaster -861 try { -862 clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper); -863 if (clusterId == null) { -864 this.abort("Cluster ID has not been set"); -865 } -866 LOG.info("ClusterId : "+clusterId); -867 } catch (KeeperException e) { -868 this.abort("Failed to retrieve Cluster ID",e); -869 } -870 -871 // In case colocated master, wait here till it's active. -872 // So backup masters won't start as regionservers. -873 // This is to avoid showing backup masters as regionservers -874 // in master web UI, or assigning any region to them. -875 waitForMasterActive(); -876 if (isStopped() || isAborted()) { -877 return; // No need for further initialization -878 } -879 -880 // watch for snapshots and other procedures -881 try { -882 rspmHost = new RegionServerProcedureManagerHost(); -883 rspmHost.loadProcedures(conf); -884 rspmHost.initialize(this); -885 } catch (KeeperException e) { -886 this.abort("Failed to reach coordination cluster when creating procedure handler.", e); -887 } -888 } -889 -890 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RV_RETURN_VALUE_IGNORED", -891 justification="We don't care about the return") -892 private void doLatch(final CountDownLatch latch) throws InterruptedException { -893 if (latch != null) { -894 // Result is ignored intentionally but if I remove the below, findbugs complains (the -895 // above justification on this method doesn't seem to suppress it). -896 boolean result = latch.await(20, TimeUnit.SECONDS); -897 } -898 } -899 -900 /** -901 * Utilty method to wait indefinitely on a znode availability while checking -902 * if the region server is shut down -903 * @param tracker znode tracker to use -904 * @throws IOException any IO exception, plus if the RS is stopped -905 * @throws InterruptedException -906 */ -907 private void blockAndCheckIfStopped(ZKNodeTracker tracker) -908 throws IOException, InterruptedException { -909 while (tracker.blockUntilAvailable(this.msgInterval, false) == null) { -910 if (this.stopped) { -911 throw new IOException("Received the shutdown message while waiting."); -912 } -913 } +823 // Setup RPC client for master communication +824 this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress( +825 this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics()); +826 } catch (Throwable t) { +827 // Call stop if error or process will stick around for ever since server +828 // puts up non-daemon threads. +829 this.rpcServices.stop(); +830 abort("Initialization of RS failed. Hence aborting RS.", t); +831 } +832 } +833 +834 /** +835 * Bring up connection to zk ensemble and then wait until a master for this cluster and then after +836 * that, wait until cluster 'up' flag has been set. This is the order in which master does things. +837 * <p> +838 * Finally open long-living server short-circuit connection. +839 */ +840 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", +841 justification="cluster Id znode read would give us correct response") +842 private void initializeZooKeeper() throws IOException, InterruptedException { +843 // Nothing to do in here if no Master in the mix. +844 if (this.masterless) { +845 return; +846 } +847 +848 // Create the master address tracker, register with zk, and start it. Then +849 // block until a master is available. No point in starting up if no master +850 // running. +851 blockAndCheckIfStopped(this.masterAddressTracker); +852 +853 // Wait on cluster being up. Master will set this flag up in zookeeper +854 // when ready. +855 blockAndCheckIfStopped(this.clusterStatusTracker); +856 +857 // If we are HMaster then the cluster id should have already been set. +858 if (clusterId == null) { +859 // Retrieve clusterId +860 // Since cluster status is now up +861 // ID should have already been set by HMaster +862 try { +863 clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper); +864 if (clusterId == null) { +865 this.abort("Cluster ID has not been set"); +866 } +867 LOG.info("ClusterId : " + clusterId); +868 } catch (KeeperException e) { +869 this.abort("Failed to retrieve Cluster ID", e); +870 } +871 } +872 +873 // In case colocated master, wait here till it's active. +874 // So backup masters won't start as regionservers. +875 // This is to avoid showing backup masters as regionservers +876 // in master web UI, or assigning any region to them. +877 waitForMasterActive(); +878 if (isStopped() || isAborted()) { +879 return; // No need for further initialization +880 } +881 +882 // watch for snapshots and other procedures +883 try { +884 rspmHost = new RegionServerProcedureManagerHost(); +885 rspmHost.loadProcedures(conf); +886 rspmHost.initialize(this); +887 } catch (KeeperException e) { +888 this.abort("Failed to reach coordination cluster when creating procedure handler.", e); +889 } +890 } +891 +892 /** +893 * Utilty method to wait indefinitely on a znode availability while checking +894 * if the region server is shut down +895 * @param tracker znode tracker to use +896 * @throws IOException any IO exception, plus if the RS is stopped +897 * @throws InterruptedException +898 */ +899 private void blockAndCheckIfStopped(ZKNodeTracker tracker) +900 throws IOException, InterruptedException { +901 while (tracker.blockUntilAvailable(this.msgInterval, false) == null) { +902 if (this.stopped) { +903 throw new IOException("Received the shutdown message while waiting."); +904 } +905 } +906 } +907 +908 /** +909 * @return True if the cluster is up. +910 */ +911 private boolean isClusterUp() { +912 return this.masterless || +913 this.clusterStatusTracker != null && this.clusterStatusTracker.isClusterUp();