Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0C055200B44 for ; Thu, 14 Jul 2016 21:46:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 07A2C160A91; Thu, 14 Jul 2016 19:46:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5A696160A8F for ; Thu, 14 Jul 2016 21:46:28 +0200 (CEST) Received: (qmail 58295 invoked by uid 500); 14 Jul 2016 19:46:25 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 57727 invoked by uid 99); 14 Jul 2016 19:46:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jul 2016 19:46:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 32E1BED103; Thu, 14 Jul 2016 19:46:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: busbey@apache.org To: commits@hbase.apache.org Date: Thu, 14 Jul 2016 19:46:33 -0000 Message-Id: <3b79bb13c85d402689586d9356b5befb@git.apache.org> In-Reply-To: <588ce14c0d8544b2906deffa43a1c0cc@git.apache.org> References: <588ce14c0d8544b2906deffa43a1c0cc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/52] [partial] hbase-site git commit: Published site at a55af38689fbe273e716ebbf6191e9515986dbf3. archived-at: Thu, 14 Jul 2016 19:46:31 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/975096b1/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html index 3ce75d2..2c621f5 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html @@ -41,2858 +41,2703 @@ 033import java.net.InetSocketAddress; 034import java.net.UnknownHostException; 035import java.util.ArrayList; -036import java.util.Arrays; -037import java.util.Collection; -038import java.util.Collections; -039import java.util.Comparator; -040import java.util.HashSet; -041import java.util.Iterator; -042import java.util.List; -043import java.util.Map; -044import java.util.Map.Entry; -045import java.util.Set; -046import java.util.concurrent.TimeUnit; -047import java.util.concurrent.atomic.AtomicInteger; -048import java.util.concurrent.atomic.AtomicReference; -049import java.util.regex.Pattern; -050 -051import javax.servlet.ServletException; -052import javax.servlet.http.HttpServlet; -053import javax.servlet.http.HttpServletRequest; -054import javax.servlet.http.HttpServletResponse; -055 -056import org.apache.commons.logging.Log; -057import org.apache.commons.logging.LogFactory; -058import org.apache.hadoop.conf.Configuration; -059import org.apache.hadoop.fs.Path; -060import org.apache.hadoop.hbase.ClusterStatus; -061import org.apache.hadoop.hbase.CoordinatedStateException; -062import org.apache.hadoop.hbase.CoordinatedStateManager; -063import org.apache.hadoop.hbase.DoNotRetryIOException; -064import org.apache.hadoop.hbase.HBaseIOException; -065import org.apache.hadoop.hbase.HBaseInterfaceAudience; -066import org.apache.hadoop.hbase.HColumnDescriptor; -067import org.apache.hadoop.hbase.HConstants; -068import org.apache.hadoop.hbase.HRegionInfo; -069import org.apache.hadoop.hbase.HTableDescriptor; -070import org.apache.hadoop.hbase.MasterNotRunningException; -071import org.apache.hadoop.hbase.MetaTableAccessor; -072import org.apache.hadoop.hbase.NamespaceDescriptor; -073import org.apache.hadoop.hbase.PleaseHoldException; -074import org.apache.hadoop.hbase.ProcedureInfo; -075import org.apache.hadoop.hbase.RegionStateListener; -076import org.apache.hadoop.hbase.ScheduledChore; -077import org.apache.hadoop.hbase.Server; -078import org.apache.hadoop.hbase.ServerLoad; -079import org.apache.hadoop.hbase.ServerName; -080import org.apache.hadoop.hbase.TableDescriptors; -081import org.apache.hadoop.hbase.TableName; -082import org.apache.hadoop.hbase.TableNotDisabledException; -083import org.apache.hadoop.hbase.TableNotFoundException; -084import org.apache.hadoop.hbase.UnknownRegionException; -085import org.apache.hadoop.hbase.classification.InterfaceAudience; -086import org.apache.hadoop.hbase.client.MasterSwitchType; -087import org.apache.hadoop.hbase.client.RegionReplicaUtil; -088import org.apache.hadoop.hbase.client.Result; -089import org.apache.hadoop.hbase.client.TableState; -090import org.apache.hadoop.hbase.coprocessor.BypassCoprocessorException; -091import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -092import org.apache.hadoop.hbase.exceptions.DeserializationException; -093import org.apache.hadoop.hbase.executor.ExecutorType; -094import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -095import org.apache.hadoop.hbase.ipc.RpcServer; -096import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -097import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; -098import org.apache.hadoop.hbase.master.balancer.BalancerChore; -099import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; -100import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; -101import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; -102import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -103import org.apache.hadoop.hbase.master.cleaner.LogCleaner; -104import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler; -105import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; -106import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; -107import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; -108import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; -109import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; -110import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure; -111import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; -112import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure; -113import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; -114import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; -115import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; -116import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; -117import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -118import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.ProcedureEvent; -119import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; -120import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; -121import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; -122import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; -123import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; -124import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -125import org.apache.hadoop.hbase.mob.MobConstants; -126import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; -127import org.apache.hadoop.hbase.monitoring.MonitoredTask; -128import org.apache.hadoop.hbase.monitoring.TaskMonitor; -129import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; -130import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; -131import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -132import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; -133import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; -134import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo; -135import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; -136import org.apache.hadoop.hbase.quotas.MasterQuotaManager; -137import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; -138import org.apache.hadoop.hbase.regionserver.HRegionServer; -139import org.apache.hadoop.hbase.regionserver.HStore; -140import org.apache.hadoop.hbase.regionserver.RSRpcServices; -141import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; -142import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; -143import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; -144import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; -145import org.apache.hadoop.hbase.replication.master.TableCFsUpdater; -146import org.apache.hadoop.hbase.replication.regionserver.Replication; -147import org.apache.hadoop.hbase.security.User; -148import org.apache.hadoop.hbase.security.UserProvider; -149import org.apache.hadoop.hbase.util.Addressing; -150import org.apache.hadoop.hbase.util.Bytes; -151import org.apache.hadoop.hbase.util.CompressionTest; -152import org.apache.hadoop.hbase.util.EncryptionTest; -153import org.apache.hadoop.hbase.util.FSUtils; -154import org.apache.hadoop.hbase.util.HFileArchiveUtil; -155import org.apache.hadoop.hbase.util.HasThread; -156import org.apache.hadoop.hbase.util.IdLock; -157import org.apache.hadoop.hbase.util.ModifyRegionUtils; -158import org.apache.hadoop.hbase.util.Pair; -159import org.apache.hadoop.hbase.util.Threads; -160import org.apache.hadoop.hbase.util.VersionInfo; -161import org.apache.hadoop.hbase.util.ZKDataMigrator; -162import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; -163import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; -164import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -165import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -166import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; -167import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; -168import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker; -169import org.apache.hadoop.hbase.zookeeper.ZKClusterId; -170import org.apache.hadoop.hbase.zookeeper.ZKUtil; -171import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -172import org.apache.zookeeper.KeeperException; -173import org.mortbay.jetty.Connector; -174import org.mortbay.jetty.nio.SelectChannelConnector; -175import org.mortbay.jetty.servlet.Context; -176 -177/** -178 * HMaster is the "master server" for HBase. An HBase cluster has one active -179 * master. If many masters are started, all compete. Whichever wins goes on to -180 * run the cluster. All others park themselves in their constructor until -181 * master or cluster shutdown or until the active master loses its lease in -182 * zookeeper. Thereafter, all running master jostle to take over master role. -183 * -184 * The Master can be asked shutdown the cluster. See {@link #shutdown()}. In -185 * this case it will tell all regionservers to go down and then wait on them -186 * all reporting in that they are down. This master will then shut itself down. -187 * -188 * You can also shutdown just this master. Call {@link #stopMaster()}. -189 * -190 * @see org.apache.zookeeper.Watcher -191 */ -192@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -193@SuppressWarnings("deprecation") -194public class HMaster extends HRegionServer implements MasterServices { -195 private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); -196 -197 /** -198 * Protection against zombie master. Started once Master accepts active responsibility and -199 * starts taking over responsibilities. Allows a finite time window before giving up ownership. -200 */ -201 private static class InitializationMonitor extends HasThread { -202 /** The amount of time in milliseconds to sleep before checking initialization status. */ -203 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout"; -204 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES); -205 -206 /** -207 * When timeout expired and initialization has not complete, call {@link System#exit(int)} when -208 * true, do nothing otherwise. -209 */ -210 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout"; -211 public static final boolean HALT_DEFAULT = false; -212 -213 private final HMaster master; -214 private final long timeout; -215 private final boolean haltOnTimeout; -216 -217 /** Creates a Thread that monitors the {@link #isInitialized()} state. */ -218 InitializationMonitor(HMaster master) { -219 super("MasterInitializationMonitor"); -220 this.master = master; -221 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT); -222 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT); -223 this.setDaemon(true); -224 } -225 -226 @Override -227 public void run() { -228 try { -229 while (!master.isStopped() && master.isActiveMaster()) { -230 Thread.sleep(timeout); -231 if (master.isInitialized()) { -232 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting."); -233 } else { -234 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please" -235 + " consider submitting a bug report including a thread dump of this process."); -236 if (haltOnTimeout) { -237 LOG.error("Zombie Master exiting. Thread dump to stdout"); -238 Threads.printThreadInfo(System.out, "Zombie HMaster"); -239 System.exit(-1); -240 } -241 } -242 } -243 } catch (InterruptedException ie) { -244 LOG.trace("InitMonitor thread interrupted. Existing."); -245 } -246 } -247 } -248 -249 // MASTER is name of the webapp and the attribute name used stuffing this -250 //instance into web context. -251 public static final String MASTER = "master"; -252 -253 // Manager and zk listener for master election -254 private final ActiveMasterManager activeMasterManager; -255 // Region server tracker -256 RegionServerTracker regionServerTracker; -257 // Draining region server tracker -258 private DrainingServerTracker drainingServerTracker; -259 // Tracker for load balancer state -260 LoadBalancerTracker loadBalancerTracker; -261 -262 // Tracker for split and merge state -263 private SplitOrMergeTracker splitOrMergeTracker; +036import java.util.Collection; +037import java.util.Collections; +038import java.util.Comparator; +039import java.util.Iterator; +040import java.util.List; +041import java.util.Map; +042import java.util.Map.Entry; +043import java.util.Set; +044import java.util.concurrent.TimeUnit; +045import java.util.concurrent.atomic.AtomicInteger; +046import java.util.concurrent.atomic.AtomicReference; +047import java.util.regex.Pattern; +048 +049import javax.servlet.ServletException; +050import javax.servlet.http.HttpServlet; +051import javax.servlet.http.HttpServletRequest; +052import javax.servlet.http.HttpServletResponse; +053 +054import org.apache.commons.logging.Log; +055import org.apache.commons.logging.LogFactory; +056import org.apache.hadoop.conf.Configuration; +057import org.apache.hadoop.fs.Path; +058import org.apache.hadoop.hbase.ClusterStatus; +059import org.apache.hadoop.hbase.CoordinatedStateException; +060import org.apache.hadoop.hbase.CoordinatedStateManager; +061import org.apache.hadoop.hbase.DoNotRetryIOException; +062import org.apache.hadoop.hbase.HBaseIOException; +063import org.apache.hadoop.hbase.HBaseInterfaceAudience; +064import org.apache.hadoop.hbase.HColumnDescriptor; +065import org.apache.hadoop.hbase.HConstants; +066import org.apache.hadoop.hbase.HRegionInfo; +067import org.apache.hadoop.hbase.HTableDescriptor; +068import org.apache.hadoop.hbase.MasterNotRunningException; +069import org.apache.hadoop.hbase.MetaTableAccessor; +070import org.apache.hadoop.hbase.NamespaceDescriptor; +071import org.apache.hadoop.hbase.PleaseHoldException; +072import org.apache.hadoop.hbase.ProcedureInfo; +073import org.apache.hadoop.hbase.RegionStateListener; +074import org.apache.hadoop.hbase.ScheduledChore; +075import org.apache.hadoop.hbase.Server; +076import org.apache.hadoop.hbase.ServerLoad; +077import org.apache.hadoop.hbase.ServerName; +078import org.apache.hadoop.hbase.TableDescriptors; +079import org.apache.hadoop.hbase.TableName; +080import org.apache.hadoop.hbase.TableNotDisabledException; +081import org.apache.hadoop.hbase.TableNotFoundException; +082import org.apache.hadoop.hbase.UnknownRegionException; +083import org.apache.hadoop.hbase.classification.InterfaceAudience; +084import org.apache.hadoop.hbase.client.MasterSwitchType; +085import org.apache.hadoop.hbase.client.Result; +086import org.apache.hadoop.hbase.client.TableState; +087import org.apache.hadoop.hbase.coprocessor.BypassCoprocessorException; +088import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +089import org.apache.hadoop.hbase.exceptions.DeserializationException; +090import org.apache.hadoop.hbase.executor.ExecutorType; +091import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +092import org.apache.hadoop.hbase.ipc.RpcServer; +093import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +094import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; +095import org.apache.hadoop.hbase.master.balancer.BalancerChore; +096import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; +097import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; +098import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +099import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +100import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +101import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler; +102import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; +103import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +104import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; +105import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; +106import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; +107import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure; +108import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; +109import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure; +110import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; +111import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; +112import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; +113import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +114import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +115import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.ProcedureEvent; +116import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; +117import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; +118import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; +119import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; +120import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; +121import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +122import org.apache.hadoop.hbase.mob.MobConstants; +123import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; +124import org.apache.hadoop.hbase.monitoring.MonitoredTask; +125import org.apache.hadoop.hbase.monitoring.TaskMonitor; +126import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; +127import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; +128import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +129import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; +130import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; +131import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo; +132import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +133import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; +134import org.apache.hadoop.hbase.regionserver.HRegionServer; +135import org.apache.hadoop.hbase.regionserver.HStore; +136import org.apache.hadoop.hbase.regionserver.RSRpcServices; +137import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +138import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; +139import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; +140import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; +141import org.apache.hadoop.hbase.replication.master.TableCFsUpdater; +142import org.apache.hadoop.hbase.replication.regionserver.Replication; +143import org.apache.hadoop.hbase.security.User; +144import org.apache.hadoop.hbase.security.UserProvider; +145import org.apache.hadoop.hbase.util.Addressing; +146import org.apache.hadoop.hbase.util.Bytes; +147import org.apache.hadoop.hbase.util.CompressionTest; +148import org.apache.hadoop.hbase.util.EncryptionTest; +149import org.apache.hadoop.hbase.util.FSUtils; +150import org.apache.hadoop.hbase.util.HFileArchiveUtil; +151import org.apache.hadoop.hbase.util.HasThread; +152import org.apache.hadoop.hbase.util.IdLock; +153import org.apache.hadoop.hbase.util.ModifyRegionUtils; +154import org.apache.hadoop.hbase.util.Pair; +155import org.apache.hadoop.hbase.util.Threads; +156import org.apache.hadoop.hbase.util.VersionInfo; +157import org.apache.hadoop.hbase.util.ZKDataMigrator; +158import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; +159import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; +160import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; +161import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; +162import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; +163import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker; +164import org.apache.hadoop.hbase.zookeeper.ZKClusterId; +165import org.apache.hadoop.hbase.zookeeper.ZKUtil; +166import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +167import org.apache.zookeeper.KeeperException; +168import org.mortbay.jetty.Connector; +169import org.mortbay.jetty.nio.SelectChannelConnector; +170import org.mortbay.jetty.servlet.Context; +171 +172/** +173 * HMaster is the "master server" for HBase. An HBase cluster has one active +174 * master. If many masters are started, all compete. Whichever wins goes on to +175 * run the cluster. All others park themselves in their constructor until +176 * master or cluster shutdown or until the active master loses its lease in +177 * zookeeper. Thereafter, all running master jostle to take over master role. +178 * +179 * The Master can be asked shutdown the cluster. See {@link #shutdown()}. In +180 * this case it will tell all regionservers to go down and then wait on them +181 * all reporting in that they are down. This master will then shut itself down. +182 * +183 * You can also shutdown just this master. Call {@link #stopMaster()}. +184 * +185 * @see org.apache.zookeeper.Watcher +186 */ +187@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +188@SuppressWarnings("deprecation") +189public class HMaster extends HRegionServer implements MasterServices { +190 private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); +191 +192 /** +193 * Protection against zombie master. Started once Master accepts active responsibility and +194 * starts taking over responsibilities. Allows a finite time window before giving up ownership. +195 */ +196 private static class InitializationMonitor extends HasThread { +197 /** The amount of time in milliseconds to sleep before checking initialization status. */ +198 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout"; +199 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES); +200 +201 /** +202 * When timeout expired and initialization has not complete, call {@link System#exit(int)} when +203 * true, do nothing otherwise. +204 */ +205 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout"; +206 public static final boolean HALT_DEFAULT = false; +207 +208 private final HMaster master; +209 private final long timeout; +210 private final boolean haltOnTimeout; +211 +212 /** Creates a Thread that monitors the {@link #isInitialized()} state. */ +213 InitializationMonitor(HMaster master) { +214 super("MasterInitializationMonitor"); +215 this.master = master; +216 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT); +217 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT); +218 this.setDaemon(true); +219 } +220 +221 @Override +222 public void run() { +223 try { +224 while (!master.isStopped() && master.isActiveMaster()) { +225 Thread.sleep(timeout); +226 if (master.isInitialized()) { +227 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting."); +228 } else { +229 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please" +230 + " consider submitting a bug report including a thread dump of this process."); +231 if (haltOnTimeout) { +232 LOG.error("Zombie Master exiting. Thread dump to stdout"); +233 Threads.printThreadInfo(System.out, "Zombie HMaster"); +234 System.exit(-1); +235 } +236 } +237 } +238 } catch (InterruptedException ie) { +239 LOG.trace("InitMonitor thread interrupted. Existing."); +240 } +241 } +242 } +243 +244 // MASTER is name of the webapp and the attribute name used stuffing this +245 //instance into web context. +246 public static final String MASTER = "master"; +247 +248 // Manager and zk listener for master election +249 private final ActiveMasterManager activeMasterManager; +250 // Region server tracker +251 RegionServerTracker regionServerTracker; +252 // Draining region server tracker +253 private DrainingServerTracker drainingServerTracker; +254 // Tracker for load balancer state +255 LoadBalancerTracker loadBalancerTracker; +256 +257 // Tracker for split and merge state +258 private SplitOrMergeTracker splitOrMergeTracker; +259 +260 // Tracker for region normalizer state +261 private RegionNormalizerTracker regionNormalizerTracker; +262 +263 private ClusterSchemaService clusterSchemaService; 264 -265 // Tracker for region normalizer state -266 private RegionNormalizerTracker regionNormalizerTracker; -267 -268 private ClusterSchemaService clusterSchemaService; -269 -270 // Metrics for the HMaster -271 final MetricsMaster metricsMaster; -272 // file system manager for the master FS operations -273 private MasterFileSystem fileSystemManager; -274 private MasterWalManager walManager; -275 -276 // server manager to deal with region server info -277 private volatile ServerManager serverManager; -278 -279 // manager of assignment nodes in zookeeper -280 private AssignmentManager assignmentManager; +265 // Metrics for the HMaster +266 final MetricsMaster metricsMaster; +267 // file system manager for the master FS operations +268 private MasterFileSystem fileSystemManager; +269 private MasterWalManager walManager; +270 +271 // server manager to deal with region server info +272 private volatile ServerManager serverManager; +273 +274 // manager of assignment nodes in zookeeper +275 private AssignmentManager assignmentManager; +276 +277 // buffer for "fatal error" notices from region servers +278 // in the cluster. This is only used for assisting +279 // operations/debugging. +280 MemoryBoundedLogMessageBuffer rsFatals; 281 -282 // buffer for "fatal error" notices from region servers -283 // in the cluster. This is only used for assisting -284 // operations/debugging. -285 MemoryBoundedLogMessageBuffer rsFatals; -286 -287 // flag set after we become the active master (used for testing) -288 private volatile boolean isActiveMaster = false; -289 -290 // flag set after we complete initialization once active, -291 // it is not private since it's used in unit tests -292 private final ProcedureEvent initialized = new ProcedureEvent("master initialized"); -293 -294 // flag set after master services are started, -295 // initialization may have not completed yet. -296 volatile boolean serviceStarted = false; -297 -298 // flag set after we complete assignMeta. -299 private final ProcedureEvent serverCrashProcessingEnabled = -300 new ProcedureEvent("server crash processing"); -301 -302 private LoadBalancer balancer; -303 private RegionNormalizer normalizer; -304 private BalancerChore balancerChore; -305 private RegionNormalizerChore normalizerChore; -306 private ClusterStatusChore clusterStatusChore; -307 private ClusterStatusPublisher clusterStatusPublisherChore = null; -308 private PeriodicDoMetrics periodicDoMetricsChore = null; -309 -310 CatalogJanitor catalogJanitorChore; -311 private LogCleaner logCleaner; -312 private HFileCleaner hfileCleaner; -313 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; -314 private MobCompactionChore mobCompactChore; -315 private MasterMobCompactionThread mobCompactThread; -316 // used to synchronize the mobCompactionStates -317 private final IdLock mobCompactionLock = new IdLock(); -318 // save the information of mob compactions in tables. -319 // the key is table name, the value is the number of compactions in that table. -320 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap(); -321 -322 MasterCoprocessorHost cpHost; +282 // flag set after we become the active master (used for testing) +283 private volatile boolean isActiveMaster = false; +284 +285 // flag set after we complete initialization once active, +286 // it is not private since it's used in unit tests +287 private final ProcedureEvent initialized = new ProcedureEvent("master initialized"); +288 +289 // flag set after master services are started, +290 // initialization may have not completed yet. +291 volatile boolean serviceStarted = false; +292 +293 // flag set after we complete assignMeta. +294 private final ProcedureEvent serverCrashProcessingEnabled = +295 new ProcedureEvent("server crash processing"); +296 +297 private LoadBalancer balancer; +298 private RegionNormalizer normalizer; +299 private BalancerChore balancerChore; +300 private RegionNormalizerChore normalizerChore; +301 private ClusterStatusChore clusterStatusChore; +302 private ClusterStatusPublisher clusterStatusPublisherChore = null; +303 private PeriodicDoMetrics periodicDoMetricsChore = null; +304 +305 CatalogJanitor catalogJanitorChore; +306 private LogCleaner logCleaner; +307 private HFileCleaner hfileCleaner; +308 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; +309 private MobCompactionChore mobCompactChore; +310 private MasterMobCompactionThread mobCompactThread; +311 // used to synchronize the mobCompactionStates +312 private final IdLock mobCompactionLock = new IdLock(); +313 // save the information of mob compactions in tables. +314 // the key is table name, the value is the number of compactions in that table. +315 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap(); +316 +317 MasterCoprocessorHost cpHost; +318 +319 private final boolean preLoadTableDescriptors; +320 +321 // Time stamps for when a hmaster became active +322 private long masterActiveTime; 323 -324 private final boolean preLoadTableDescriptors; -325 -326 // Time stamps for when a hmaster became active -327 private long masterActiveTime; -328 -329 //should we check the compression codec type at master side, default true, HBASE-6370 -330 private final boolean masterCheckCompression; +324 //should we check the compression codec type at master side, default true, HBASE-6370 +325 private final boolean masterCheckCompression; +326 +327 //should we check encryption settings at master side, default true +328 private final boolean masterCheckEncryption; +329 +330 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); 331 -332 //should we check encryption settings at master side, default true -333 private final boolean masterCheckEncryption; -334 -335 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); +332 // monitor for snapshot of hbase tables +333 SnapshotManager snapshotManager; +334 // monitor for distributed procedures +335 private MasterProcedureManagerHost mpmHost; 336 -337 // monitor for snapshot of hbase tables -338 SnapshotManager snapshotManager; -339 // monitor for distributed procedures -340 private MasterProcedureManagerHost mpmHost; -341 -342 // it is assigned after 'initialized' guard set to true, so should be volatile -343 private volatile MasterQuotaManager quotaManager; -344 -345 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; -346 private WALProcedureStore procedureStore; -347 -348 // handle table states -349 private TableStateManager tableStateManager; -350 -351 private long splitPlanCount; -352 private long mergePlanCount; -353 -354 /** flag used in test cases in order to simulate RS failures during master initialization */ -355 private volatile boolean initializationBeforeMetaAssignment = false; -356 -357 /** jetty server for master to redirect requests to regionserver infoServer */ -358 private org.mortbay.jetty.Server masterJettyServer; -359 -360 public static class RedirectServlet extends HttpServlet { -361 private static final long serialVersionUID = 2894774810058302472L; -362 private static int regionServerInfoPort; -363 -364 @Override -365 public void doGet(HttpServletRequest request, -366 HttpServletResponse response) throws ServletException, IOException { -367 String redirectUrl = request.getScheme() + "://" -368 + request.getServerName() + ":" + regionServerInfoPort -369 + request.getRequestURI(); -370 response.sendRedirect(redirectUrl); -371 } -372 } -373 -374 private static class PeriodicDoMetrics extends ScheduledChore { -375 private final HMaster server; -376 public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { -377 super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); -378 this.server = server; +337 // it is assigned after 'initialized' guard set to true, so should be volatile +338 private volatile MasterQuotaManager quotaManager; +339 +340 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; +341 private WALProcedureStore procedureStore; +342 +343 // handle table states +344 private TableStateManager tableStateManager; +345 +346 private long splitPlanCount; +347 private long mergePlanCount; +348 +349 /** flag used in test cases in order to simulate RS failures during master initialization */ +350 private volatile boolean initializationBeforeMetaAssignment = false; +351 +352 /** jetty server for master to redirect requests to regionserver infoServer */ +353 private org.mortbay.jetty.Server masterJettyServer; +354 +355 public static class RedirectServlet extends HttpServlet { +356 private static final long serialVersionUID = 2894774810058302472L; +357 private static int regionServerInfoPort; +358 +359 @Override +360 public void doGet(HttpServletRequest request, +361 HttpServletResponse response) throws ServletException, IOException { +362 String redirectUrl = request.getScheme() + "://" +363 + request.getServerName() + ":" + regionServerInfoPort +364 + request.getRequestURI(); +365 response.sendRedirect(redirectUrl); +366 } +367 } +368 +369 private static class PeriodicDoMetrics extends ScheduledChore { +370 private final HMaster server; +371 public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) { +372 super(server.getServerName() + "-DoMetricsChore", server, doMetricsInterval); +373 this.server = server; +374 } +375 +376 @Override +377 protected void chore() { +378 server.doMetrics(); 379 } -380 -381 @Override -382 protected void chore() { -383 server.doMetrics(); -384 } -385 } -386 -387 /** -388 * Initializes the HMaster. The steps are as follows: +380 } +381 +382 /** +383 * Initializes the HMaster. The steps are as follows: +384 * +385 * <ol> +386 * <li>Initialize the local HRegionServer +387 * <li>Start the ActiveMasterManager. +388 * </ol> 389 * -390 * <ol> -391 * <li>Initialize the local HRegionServer -392 * <li>Start the ActiveMasterManager. -393 * </ol> -394 * -395 * Remaining steps of initialization occur in -396 * #finishActiveMasterInitialization(MonitoredTask) after -397 * the master becomes the active one. -398 */ -399 public HMaster(final Configuration conf, CoordinatedStateManager csm) -400 throws IOException, KeeperException { -401 super(conf, csm); -402 this.rsFatals = new MemoryBoundedLogMessageBuffer( -403 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024)); -404 -405 LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) + -406 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false)); +390 * Remaining steps of initialization occur in +391 * #finishActiveMasterInitialization(MonitoredTask) after +392 * the master becomes the active one. +393 */ +394 public HMaster(final Configuration conf, CoordinatedStateManager csm) +395 throws IOException, KeeperException { +396 super(conf, csm); +397 this.rsFatals = new MemoryBoundedLogMessageBuffer( +398 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024)); +399 +400 LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) + +401 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false)); +402 +403 // Disable usage of meta replicas in the master +404 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); +405 +406 Replication.decorateMasterConfiguration(this.conf); 407 -408 // Disable usage of meta replicas in the master -409 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); -410 -411 Replication.decorateMasterConfiguration(this.conf); -412 -413 // Hack! Maps DFSClient => Master for logs. HDFS made this -414 // config param for task trackers, but we can piggyback off of it. -415 if (this.conf.get("mapreduce.task.attempt.id") == null) { -416 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString()); -417 } -418 -419 // should we check the compression codec type at master side, default true, HBASE-6370 -420 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true); +408 // Hack! Maps DFSClient => Master for logs. HDFS made this +409 // config param for task trackers, but we can piggyback off of it. +410 if (this.conf.get("mapreduce.task.attempt.id") == null) { +411 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString()); +412 } +413 +414 // should we check the compression codec type at master side, default true, HBASE-6370 +415 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true); +416 +417 // should we check encryption settings at master side, default true +418 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true); +419 +420 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); 421 -422 // should we check encryption settings at master side, default true -423 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true); +422 // preload table descriptor at startup +423 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); 424 -425 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); +425 // Do we publish the status? 426 -427 // preload table descriptor at startup -428 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); -429 -430 // Do we publish the status? -431 -432 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, -433 HConstants.STATUS_PUBLISHED_DEFAULT); -434 Class<? extends ClusterStatusPublisher.Publisher> publisherClass = -435 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS, -436 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS, -437 ClusterStatusPublisher.Publisher.class); -438 -439 if (shouldPublish) { -440 if (publisherClass == null) { -441 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " + -442 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS + -443 " is not set - not publishing status"); -444 } else { -445 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass); -446 getChoreService().scheduleChore(clusterStatusPublisherChore); -447 } -448 } -449 -450 // Some unit tests don't need a cluster, so no zookeeper at all -451 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -452 activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); -453 int infoPort = putUpJettyServer(); -454 startActiveMasterManager(infoPort); -455 } else { -456 activeMasterManager = null; -457 } -458 } -459 -460 // return the actual infoPort, -1 means disable info server. -461 private int putUpJettyServer() throws IOException { -462 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { -463 return -1; -464 } -465 int infoPort = conf.getInt("hbase.master.info.port.orig", -466 HConstants.DEFAULT_MASTER_INFOPORT); -467 // -1 is for disabling info server, so no redirecting -468 if (infoPort < 0 || infoServer == null) { -469 return -1; -470 } -471 String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); -472 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) { -473 String msg = -474 "Failed to start redirecting jetty server. Address " + addr -475 + " does not belong to this host. Correct configuration parameter: " -476 + "hbase.master.info.bindAddress"; -477 LOG.error(msg); -478 throw new IOException(msg); +427 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, +428 HConstants.STATUS_PUBLISHED_DEFAULT); +429 Class<? extends ClusterStatusPublisher.Publisher> publisherClass = +430 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS, +431 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS, +432 ClusterStatusPublisher.Publisher.class); +433 +434 if (shouldPublish) {