From commits-return-65387-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Sat Jan 13 16:32:00 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 2D2A618078F for ; Sat, 13 Jan 2018 16:32:00 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 1C36D160C4B; Sat, 13 Jan 2018 15:32:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D4926160C4C for ; Sat, 13 Jan 2018 16:31:57 +0100 (CET) Received: (qmail 87437 invoked by uid 500); 13 Jan 2018 15:31:55 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 87278 invoked by uid 99); 13 Jan 2018 15:31:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Jan 2018 15:31:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7997AF3324; Sat, 13 Jan 2018 15:31:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 13 Jan 2018 15:32:17 -0000 Message-Id: <591919ae26724164b47a73548333b0bc@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [28/47] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/554d61c6/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html index 1fc3ca7..ec25b93 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html @@ -62,3477 +62,3486 @@ 054import javax.servlet.http.HttpServletResponse; 055import org.apache.hadoop.conf.Configuration; 056import org.apache.hadoop.fs.Path; -057import org.apache.hadoop.hbase.ClusterMetrics; -058import org.apache.hadoop.hbase.ClusterMetrics.Option; -059import org.apache.hadoop.hbase.ClusterMetricsBuilder; -060import org.apache.hadoop.hbase.CoordinatedStateException; -061import org.apache.hadoop.hbase.DoNotRetryIOException; -062import org.apache.hadoop.hbase.HBaseIOException; -063import org.apache.hadoop.hbase.HBaseInterfaceAudience; -064import org.apache.hadoop.hbase.HConstants; -065import org.apache.hadoop.hbase.InvalidFamilyOperationException; -066import org.apache.hadoop.hbase.MasterNotRunningException; -067import org.apache.hadoop.hbase.MetaTableAccessor; -068import org.apache.hadoop.hbase.NamespaceDescriptor; -069import org.apache.hadoop.hbase.PleaseHoldException; -070import org.apache.hadoop.hbase.ReplicationPeerNotFoundException; -071import org.apache.hadoop.hbase.ServerLoad; -072import org.apache.hadoop.hbase.ServerMetricsBuilder; -073import org.apache.hadoop.hbase.ServerName; -074import org.apache.hadoop.hbase.TableDescriptors; -075import org.apache.hadoop.hbase.TableName; -076import org.apache.hadoop.hbase.TableNotDisabledException; -077import org.apache.hadoop.hbase.TableNotFoundException; -078import org.apache.hadoop.hbase.UnknownRegionException; -079import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -080import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -081import org.apache.hadoop.hbase.client.MasterSwitchType; -082import org.apache.hadoop.hbase.client.RegionInfo; -083import org.apache.hadoop.hbase.client.Result; -084import org.apache.hadoop.hbase.client.TableDescriptor; -085import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -086import org.apache.hadoop.hbase.client.TableState; -087import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -088import org.apache.hadoop.hbase.exceptions.DeserializationException; -089import org.apache.hadoop.hbase.exceptions.MergeRegionException; -090import org.apache.hadoop.hbase.executor.ExecutorType; -091import org.apache.hadoop.hbase.favored.FavoredNodesManager; -092import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; -093import org.apache.hadoop.hbase.http.InfoServer; -094import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -095import org.apache.hadoop.hbase.ipc.RpcServer; -096import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -097import org.apache.hadoop.hbase.log.HBaseMarkers; -098import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; -099import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -100import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure; -101import org.apache.hadoop.hbase.master.assignment.RegionStates; -102import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode; -103import org.apache.hadoop.hbase.master.balancer.BalancerChore; -104import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; -105import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; -106import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; -107import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -108import org.apache.hadoop.hbase.master.cleaner.LogCleaner; -109import org.apache.hadoop.hbase.master.cleaner.ReplicationMetaCleaner; -110import org.apache.hadoop.hbase.master.locking.LockManager; -111import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; -112import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; -113import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; -114import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; -115import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; -116import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; -117import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; -118import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; -119import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; -120import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; -121import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -122import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; -123import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; -124import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; -125import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; -126import org.apache.hadoop.hbase.master.procedure.RecoverMetaProcedure; -127import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; -128import org.apache.hadoop.hbase.master.replication.AddPeerProcedure; -129import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure; -130import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure; -131import org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure; -132import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure; -133import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager; -134import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure; -135import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -136import org.apache.hadoop.hbase.mob.MobConstants; -137import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; -138import org.apache.hadoop.hbase.monitoring.MonitoredTask; -139import org.apache.hadoop.hbase.monitoring.TaskMonitor; -140import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; -141import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; -142import org.apache.hadoop.hbase.procedure2.LockedResource; -143import org.apache.hadoop.hbase.procedure2.Procedure; -144import org.apache.hadoop.hbase.procedure2.ProcedureEvent; -145import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -146import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure; -147import org.apache.hadoop.hbase.procedure2.RemoteProcedureException; -148import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; -149import org.apache.hadoop.hbase.quotas.MasterQuotaManager; -150import org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver; -151import org.apache.hadoop.hbase.quotas.QuotaObserverChore; -152import org.apache.hadoop.hbase.quotas.QuotaUtil; -153import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore; -154import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifier; -155import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshotNotifierFactory; -156import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; -157import org.apache.hadoop.hbase.regionserver.HRegionServer; -158import org.apache.hadoop.hbase.regionserver.HStore; -159import org.apache.hadoop.hbase.regionserver.RSRpcServices; -160import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; -161import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; -162import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; -163import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; -164import org.apache.hadoop.hbase.replication.ReplicationException; -165import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -166import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; -167import org.apache.hadoop.hbase.replication.master.ReplicationPeerConfigUpgrader; -168import org.apache.hadoop.hbase.replication.regionserver.Replication; -169import org.apache.hadoop.hbase.security.AccessDeniedException; -170import org.apache.hadoop.hbase.security.UserProvider; -171import org.apache.hadoop.hbase.trace.TraceUtil; -172import org.apache.hadoop.hbase.util.Addressing; -173import org.apache.hadoop.hbase.util.Bytes; -174import org.apache.hadoop.hbase.util.CompressionTest; -175import org.apache.hadoop.hbase.util.EncryptionTest; -176import org.apache.hadoop.hbase.util.HFileArchiveUtil; -177import org.apache.hadoop.hbase.util.HasThread; -178import org.apache.hadoop.hbase.util.IdLock; -179import org.apache.hadoop.hbase.util.ModifyRegionUtils; -180import org.apache.hadoop.hbase.util.Pair; -181import org.apache.hadoop.hbase.util.Threads; -182import org.apache.hadoop.hbase.util.VersionInfo; -183import org.apache.hadoop.hbase.util.ZKDataMigrator; -184import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; -185import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -186import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker; -187import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; -188import org.apache.hadoop.hbase.zookeeper.ZKClusterId; -189import org.apache.hadoop.hbase.zookeeper.ZKUtil; -190import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -191import org.apache.hadoop.hbase.zookeeper.ZNodePaths; -192import org.apache.yetus.audience.InterfaceAudience; -193import org.apache.zookeeper.KeeperException; -194import org.eclipse.jetty.server.Server; -195import org.eclipse.jetty.server.ServerConnector; -196import org.eclipse.jetty.servlet.ServletHolder; -197import org.eclipse.jetty.webapp.WebAppContext; -198import org.slf4j.Logger; -199import org.slf4j.LoggerFactory; -200 -201import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -202import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -203import org.apache.hbase.thirdparty.com.google.common.collect.Maps; -204 -205import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -206import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; -207import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServerInfo; -208import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; -209import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy; -210import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -211import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; -212 -213/** -214 * HMaster is the "master server" for HBase. An HBase cluster has one active -215 * master. If many masters are started, all compete. Whichever wins goes on to -216 * run the cluster. All others park themselves in their constructor until -217 * master or cluster shutdown or until the active master loses its lease in -218 * zookeeper. Thereafter, all running master jostle to take over master role. -219 * -220 * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}. In -221 * this case it will tell all regionservers to go down and then wait on them -222 * all reporting in that they are down. This master will then shut itself down. -223 * -224 * <p>You can also shutdown just this master. Call {@link #stopMaster()}. -225 * -226 * @see org.apache.zookeeper.Watcher -227 */ -228@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -229@SuppressWarnings("deprecation") -230public class HMaster extends HRegionServer implements MasterServices { -231 private static Logger LOG = LoggerFactory.getLogger(HMaster.class.getName()); -232 -233 /** -234 * Protection against zombie master. Started once Master accepts active responsibility and -235 * starts taking over responsibilities. Allows a finite time window before giving up ownership. -236 */ -237 private static class InitializationMonitor extends HasThread { -238 /** The amount of time in milliseconds to sleep before checking initialization status. */ -239 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout"; -240 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES); -241 -242 /** -243 * When timeout expired and initialization has not complete, call {@link System#exit(int)} when -244 * true, do nothing otherwise. -245 */ -246 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout"; -247 public static final boolean HALT_DEFAULT = false; -248 -249 private final HMaster master; -250 private final long timeout; -251 private final boolean haltOnTimeout; -252 -253 /** Creates a Thread that monitors the {@link #isInitialized()} state. */ -254 InitializationMonitor(HMaster master) { -255 super("MasterInitializationMonitor"); -256 this.master = master; -257 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT); -258 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT); -259 this.setDaemon(true); -260 } -261 -262 @Override -263 public void run() { -264 try { -265 while (!master.isStopped() && master.isActiveMaster()) { -266 Thread.sleep(timeout); -267 if (master.isInitialized()) { -268 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting."); -269 } else { -270 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please" -271 + " consider submitting a bug report including a thread dump of this process."); -272 if (haltOnTimeout) { -273 LOG.error("Zombie Master exiting. Thread dump to stdout"); -274 Threads.printThreadInfo(System.out, "Zombie HMaster"); -275 System.exit(-1); -276 } -277 } -278 } -279 } catch (InterruptedException ie) { -280 LOG.trace("InitMonitor thread interrupted. Existing."); -281 } -282 } -283 } -284 -285 // MASTER is name of the webapp and the attribute name used stuffing this -286 //instance into web context. -287 public static final String MASTER = "master"; -288 -289 // Manager and zk listener for master election -290 private final ActiveMasterManager activeMasterManager; -291 // Region server tracker -292 RegionServerTracker regionServerTracker; -293 // Draining region server tracker -294 private DrainingServerTracker drainingServerTracker; -295 // Tracker for load balancer state -296 LoadBalancerTracker loadBalancerTracker; -297 -298 // Tracker for split and merge state -299 private SplitOrMergeTracker splitOrMergeTracker; -300 -301 // Tracker for region normalizer state -302 private RegionNormalizerTracker regionNormalizerTracker; -303 -304 //Tracker for master maintenance mode setting -305 private MasterMaintenanceModeTracker maintenanceModeTracker; -306 -307 private ClusterSchemaService clusterSchemaService; -308 -309 public static final String HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS = -310 "hbase.master.wait.on.service.seconds"; -311 public static final int DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS = 5 * 60; -312 -313 // Metrics for the HMaster -314 final MetricsMaster metricsMaster; -315 // file system manager for the master FS operations -316 private MasterFileSystem fileSystemManager; -317 private MasterWalManager walManager; -318 -319 // server manager to deal with region server info -320 private volatile ServerManager serverManager; -321 -322 // manager of assignment nodes in zookeeper -323 private AssignmentManager assignmentManager; -324 -325 // manager of replication -326 private ReplicationPeerManager replicationPeerManager; -327 -328 // buffer for "fatal error" notices from region servers -329 // in the cluster. This is only used for assisting -330 // operations/debugging. -331 MemoryBoundedLogMessageBuffer rsFatals; -332 -333 // flag set after we become the active master (used for testing) -334 private volatile boolean activeMaster = false; -335 -336 // flag set after we complete initialization once active -337 private final ProcedureEvent<?> initialized = new ProcedureEvent<>("master initialized"); -338 -339 // flag set after master services are started, -340 // initialization may have not completed yet. -341 volatile boolean serviceStarted = false; -342 -343 // flag set after we complete assignMeta. -344 private final ProcedureEvent<?> serverCrashProcessingEnabled = -345 new ProcedureEvent<>("server crash processing"); -346 -347 // Maximum time we should run balancer for -348 private final int maxBlancingTime; -349 // Maximum percent of regions in transition when balancing -350 private final double maxRitPercent; -351 -352 private final LockManager lockManager = new LockManager(this); -353 -354 private LoadBalancer balancer; -355 private RegionNormalizer normalizer; -356 private BalancerChore balancerChore; -357 private RegionNormalizerChore normalizerChore; -358 private ClusterStatusChore clusterStatusChore; -359 private ClusterStatusPublisher clusterStatusPublisherChore = null; -360 -361 CatalogJanitor catalogJanitorChore; -362 private ReplicationMetaCleaner replicationMetaCleaner; -363 private LogCleaner logCleaner; -364 private HFileCleaner hfileCleaner; -365 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; -366 private MobCompactionChore mobCompactChore; -367 private MasterMobCompactionThread mobCompactThread; -368 // used to synchronize the mobCompactionStates -369 private final IdLock mobCompactionLock = new IdLock(); -370 // save the information of mob compactions in tables. -371 // the key is table name, the value is the number of compactions in that table. -372 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap(); -373 -374 MasterCoprocessorHost cpHost; -375 -376 private final boolean preLoadTableDescriptors; -377 -378 // Time stamps for when a hmaster became active -379 private long masterActiveTime; -380 -381 // Time stamp for when HMaster finishes becoming Active Master -382 private long masterFinishedInitializationTime; -383 -384 //should we check the compression codec type at master side, default true, HBASE-6370 -385 private final boolean masterCheckCompression; -386 -387 //should we check encryption settings at master side, default true -388 private final boolean masterCheckEncryption; -389 -390 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); -391 -392 // monitor for snapshot of hbase tables -393 SnapshotManager snapshotManager; -394 // monitor for distributed procedures -395 private MasterProcedureManagerHost mpmHost; -396 -397 // it is assigned after 'initialized' guard set to true, so should be volatile -398 private volatile MasterQuotaManager quotaManager; -399 private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier; -400 private QuotaObserverChore quotaObserverChore; -401 private SnapshotQuotaObserverChore snapshotQuotaChore; -402 -403 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; -404 private WALProcedureStore procedureStore; -405 -406 // handle table states -407 private TableStateManager tableStateManager; -408 -409 private long splitPlanCount; -410 private long mergePlanCount; -411 -412 /* Handle favored nodes information */ -413 private FavoredNodesManager favoredNodesManager; -414 -415 /** jetty server for master to redirect requests to regionserver infoServer */ -416 private Server masterJettyServer; -417 -418 public static class RedirectServlet extends HttpServlet { -419 private static final long serialVersionUID = 2894774810058302473L; -420 private final int regionServerInfoPort; -421 private final String regionServerHostname; -422 -423 /** -424 * @param infoServer that we're trying to send all requests to -425 * @param hostname may be null. if given, will be used for redirects instead of host from client. -426 */ -427 public RedirectServlet(InfoServer infoServer, String hostname) { -428 regionServerInfoPort = infoServer.getPort(); -429 regionServerHostname = hostname; -430 } -431 -432 @Override -433 public void doGet(HttpServletRequest request, -434 HttpServletResponse response) throws ServletException, IOException { -435 String redirectHost = regionServerHostname; -436 if(redirectHost == null) { -437 redirectHost = request.getServerName(); -438 if(!Addressing.isLocalAddress(InetAddress.getByName(redirectHost))) { -439 LOG.warn("Couldn't resolve '" + redirectHost + "' as an address local to this node and '" + -440 MASTER_HOSTNAME_KEY + "' is not set; client will get a HTTP 400 response. If " + -441 "your HBase deployment relies on client accessible names that the region server process " + -442 "can't resolve locally, then you should set the previously mentioned configuration variable " + -443 "to an appropriate hostname."); -444 // no sending client provided input back to the client, so the goal host is just in the logs. -445 response.sendError(400, "Request was to a host that I can't resolve for any of the network interfaces on " + -446 "this node. If this is due to an intermediary such as an HTTP load balancer or other proxy, your HBase " + -447 "administrator can set '" + MASTER_HOSTNAME_KEY + "' to point to the correct hostname."); -448 return; -449 } -450 } -451 // TODO this scheme should come from looking at the scheme registered in the infoserver's http server for the -452 // host and port we're using, but it's buried way too deep to do that ATM. -453 String redirectUrl = request.getScheme() + "://" -454 + redirectHost + ":" + regionServerInfoPort -455 + request.getRequestURI(); -456 response.sendRedirect(redirectUrl); -457 } -458 } -459 -460 /** -461 * Initializes the HMaster. The steps are as follows: -462 * <p> -463 * <ol> -464 * <li>Initialize the local HRegionServer -465 * <li>Start the ActiveMasterManager. -466 * </ol> -467 * <p> -468 * Remaining steps of initialization occur in -469 * #finishActiveMasterInitialization(MonitoredTask) after -470 * the master becomes the active one. -471 */ -472 public HMaster(final Configuration conf) -473 throws IOException, KeeperException { -474 super(conf); -475 TraceUtil.initTracer(conf); -476 try { -477 this.rsFatals = new MemoryBoundedLogMessageBuffer( -478 conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024)); -479 LOG.info("hbase.rootdir=" + getRootDir() + -480 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false)); -481 -482 // Disable usage of meta replicas in the master -483 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); -484 -485 Replication.decorateMasterConfiguration(this.conf); -486 -487 // Hack! Maps DFSClient => Master for logs. HDFS made this -488 // config param for task trackers, but we can piggyback off of it. -489 if (this.conf.get("mapreduce.task.attempt.id") == null) { -490 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString()); -491 } -492 -493 // should we check the compression codec type at master side, default true, HBASE-6370 -494 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true); -495 -496 // should we check encryption settings at master side, default true -497 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true); -498 -499 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); -500 -501 // preload table descriptor at startup -502 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); -503 -504 this.maxBlancingTime = getMaxBalancingTime(); -505 this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT, -506 HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT); -507 -508 // Do we publish the status? -509 -510 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, -511 HConstants.STATUS_PUBLISHED_DEFAULT); -512 Class<? extends ClusterStatusPublisher.Publisher> publisherClass = -513 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS, -514 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS, -515 ClusterStatusPublisher.Publisher.class); -516 -517 if (shouldPublish) { -518 if (publisherClass == null) { -519 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " + -520 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS + -521 " is not set - not publishing status"); -522 } else { -523 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass); -524 getChoreService().scheduleChore(clusterStatusPublisherChore); -525 } -526 } -527 -528 // Some unit tests don't need a cluster, so no zookeeper at all -529 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -530 this.activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); -531 } else { -532 this.activeMasterManager = null; -533 } -534 } catch (Throwable t) { -535 // Make sure we log the exception. HMaster is often started via reflection and the -536 // cause of failed startup is lost. -537 LOG.error("Failed construction of Master", t); -538 throw t; -539 } -540 } -541 -542 // Main run loop. Calls through to the regionserver run loop AFTER becoming active Master; will -543 // block in here until then. -544 @Override -545 public void run() { -546 try { -547 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -548 try { -549 int infoPort = putUpJettyServer(); -550 startActiveMasterManager(infoPort); -551 } catch (Throwable t) { -552 // Make sure we log the exception. -553 String error = "Failed to become Active Master"; -554 LOG.error(error, t); -555 // Abort should have been called already. -556 if (!isAborted()) { -557 abort(error, t); -558 } -559 } -560 } -561 // Fall in here even if we have been aborted. Need to run the shutdown services and -562 // the super run call will do this for us. -563 super.run(); -564 } finally { -565 if (this.clusterSchemaService != null) { -566 // If on way out, then we are no longer active master. -567 this.clusterSchemaService.stopAsync(); -568 try { -569 this.clusterSchemaService.awaitTerminated( -570 getConfiguration().getInt(HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS, -571 DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS); -572 } catch (TimeoutException te) { -573 LOG.warn("Failed shutdown of clusterSchemaService", te); -574 } -575 } -576 this.activeMaster = false; -577 } -578 } -579 -580 // return the actual infoPort, -1 means disable info server. -581 private int putUpJettyServer() throws IOException { -582 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { -583 return -1; -584 } -585 final int infoPort = conf.getInt("hbase.master.info.port.orig", -586 HConstants.DEFAULT_MASTER_INFOPORT); -587 // -1 is for disabling info server, so no redirecting -588 if (infoPort < 0 || infoServer == null) { -589 return -1; -590 } -591 if(infoPort == infoServer.getPort()) { -592 return infoPort; -593 } -594 final String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); -595 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) { -596 String msg = -597 "Failed to start redirecting jetty server. Address " + addr -598 + " does not belong to this host. Correct configuration parameter: " -599 + "hbase.master.info.bindAddress"; -600 LOG.error(msg); -601 throw new IOException(msg); -602 } -603 -604 // TODO I'm pretty sure we could just add another binding to the InfoServer run by -605 // the RegionServer and have it run the RedirectServlet instead of standing up -606 // a second entire stack here. -607 masterJettyServer = new Server(); -608 final ServerConnector connector = new ServerConnector(masterJettyServer); -609 connector.setHost(addr); -610 connector.setPort(infoPort); -611 masterJettyServer.addConnector(connector); -612 masterJettyServer.setStopAtShutdown(true); -613 -614 final String redirectHostname = shouldUseThisHostnameInstead() ? useThisHostnameInstead : null; -615 -616 final RedirectServlet redirect = new RedirectServlet(infoServer, redirectHostname); -617 final WebAppContext context = new WebAppContext(null, "/", null, null, null, null, WebAppContext.NO_SESSIONS); -618 context.addServlet(new ServletHolder(redirect), "/*"); -619 context.setServer(masterJettyServer); -620 -621 try { -622 masterJettyServer.start(); -623 } catch (Exception e) { -624 throw new IOException("Failed to start redirecting jetty server", e); -625 } -626 return connector.getLocalPort(); -627 } -628 -629 @Override -630 protected Function<TableDescriptorBuilder, TableDescriptorBuilder> getMetaTableObserver() { -631 return builder -> builder.setRegionReplication(conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM)); -632 } -633 /** -634 * For compatibility, if failed with regionserver credentials, try the master one -635 */ -636 @Override -637 protected void login(UserProvider user, String host) throws IOException { -638 try { -639 super.login(user, host); -640 } catch (IOException ie) { -641 user.login("hbase.master.keytab.file", -642 "hbase.master.kerberos.principal", host); -643 } -644 } -645 -646 /** -647 * If configured to put regions on active master, -648 * wait till a backup master becomes active. -649 * Otherwise, loop till the server is stopped or aborted. -650 */ -651 @Override -652 protected void waitForMasterActive(){ -653 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf); -654 while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) { -655 sleeper.sleep(); -656 } -657 } -658 -659 @VisibleForTesting -660 public MasterRpcServices getMasterRpcServices() { -661 return (MasterRpcServices)rpcServices; -662 } -663 -664 public boolean balanceSwitch(final boolean b) throws IOException { -665 return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC); -666 } -667 -668 @Override -669 protected String getProcessName() { -670 return MASTER; -671 } -672 -673 @Override -674 protected boolean canCreateBaseZNode() { -675 return true; -676 } -677 -678 @Override -679 protected boolean canUpdateTableDescriptor() { -680 return true; -681 } -682 -683 @Override -684 protected RSRpcServices createRpcServices() throws IOException { -685 return new MasterRpcServices(this); -686 } -687 -688 @Override -689 protected void configureInfoServer() { -690 infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class); -691 infoServer.setAttribute(MASTER, this); -692 if (LoadBalancer.isTablesOnMaster(conf)) { -693 super.configureInfoServer(); -694 } -695 } -696 -697 @Override -698 protected Class<? extends HttpServlet> getDumpServlet() { -699 return MasterDumpServlet.class; -700 } -701 -702 @Override -703 public MetricsMaster getMasterMetrics() { -704 return metricsMaster; -705 } -706 -707 /** -708 * Initialize all ZK based system trackers. -709 */ -710 void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, KeeperException, -711 CoordinatedStateException, ReplicationException { -712 this.balancer = LoadBalancerFactory.getLoadBalancer(conf); -713 this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf); -714 this.normalizer.setMasterServices(this); -715 this.normalizer.setMasterRpcServices((MasterRpcServices)rpcServices); -716 this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this); -717 this.loadBalancerTracker.start(); -718 -719 this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this); -720 this.regionNormalizerTracker.start(); -721 -722 this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this); -723 this.splitOrMergeTracker.start(); -724 -725 // Create Assignment Manager -726 this.assignmentManager = new AssignmentManager(this); -727 this.assignmentManager.start(); -728 -729 this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf); -730 -731 this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager); -732 this.regionServerTracker.start(); -733 -734 this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager); -735 this.drainingServerTracker.start(); -736 -737 this.maintenanceModeTracker = new MasterMaintenanceModeTracker(zooKeeper); -738 this.maintenanceModeTracker.start(); -739 -740 // Set the cluster as up. If new RSs, they'll be waiting on this before -741 // going ahead with their startup. -742 boolean wasUp = this.clusterStatusTracker.isClusterUp(); -743 if (!wasUp) this.clusterStatusTracker.setClusterUp(); -744 -745 LOG.info("Server active/primary master=" + this.serverName + -746 ", sessionid=0x" + -747 Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) + -748 ", setting cluster-up flag (Was=" + wasUp + ")"); -749 -750 // create/initialize the snapshot manager and other procedure managers -751 this.snapshotManager = new SnapshotManager(); -752 this.mpmHost = new MasterProcedureManagerHost(); -753 this.mpmHost.register(this.snapshotManager); -754 this.mpmHost.register(new MasterFlushTableProcedureManager()); -755 this.mpmHost.loadProcedures(conf); -756 this.mpmHost.initialize(this, this.metricsMaster); -757 } -758 -759 /** -760 * Finish initialization of HMaster after becoming the primary master. -761 * -762 * <ol> -763 * <li>Initialize master components - file system manager, server manager, -764 * assignment manager, region server tracker, etc</li> -765 * <li>Start necessary service threads - balancer, catalog janior, -766 * executor services, etc</li> -767 * <li>Set cluster as UP in ZooKeeper</li> -768 * <li>Wait for RegionServers to check-in</li> -769 * <li>Split logs and perform data recovery, if necessary</li> -770 * <li>Ensure assignment of meta/namespace regions<li> -771 * <li>Handle either fresh cluster start or master failover</li> -772 * </ol> -773 */ -774 private void finishActiveMasterInitialization(MonitoredTask status) throws IOException, -775 InterruptedException, KeeperException, CoordinatedStateException, ReplicationException { -776 Thread zombieDetector = new Thread(new InitializationMonitor(this), -777 "ActiveMasterInitializationMonitor-" + System.currentTimeMillis()); -778 zombieDetector.setDaemon(true); -779 zombieDetector.start(); -780 -781 /* -782 * We are active master now... go initialize components we need to run. -783 */ -784 status.setStatus("Initializing Master file system"); -785 -786 this.masterActiveTime = System.currentTimeMillis(); -787 // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring. -788 // Initialize the chunkCreator -789 initializeMemStoreChunkCreator(); -790 this.fileSystemManager = new MasterFileSystem(this); -791 this.walManager = new MasterWalManager(this); -792 -793 // enable table descriptors cache -794 this.tableDescriptors.setCacheOn(); -795 -796 // warm-up HTDs cache on master initialization -797 if (preLoadTableDescriptors) { -798 status.setStatus("Pre-loading table descriptors"); -799 this.tableDescriptors.getAll(); -800 } -801 -802 // Publish cluster ID -803 status.setStatus("Publishing Cluster ID in ZooKeeper"); -804 ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId()); -805 -806 this.serverManager = createServerManager(this); -807 -808 this.tableStateManager = new TableStateManager(this); -809 -810 status.setStatus("Initializing ZK system trackers"); -811 initializeZKBasedSystemTrackers(); +057import org.apache.hadoop.hbase.ClusterId; +058import org.apache.hadoop.hbase.ClusterMetrics; +059import org.apache.hadoop.hbase.ClusterMetrics.Option; +060import org.apache.hadoop.hbase.ClusterMetricsBuilder; +061import org.apache.hadoop.hbase.CoordinatedStateException; +062import org.apache.hadoop.hbase.DoNotRetryIOException; +063import org.apache.hadoop.hbase.HBaseIOException; +064import org.apache.hadoop.hbase.HBaseInterfaceAudience; +065import org.apache.hadoop.hbase.HConstants; +066import org.apache.hadoop.hbase.InvalidFamilyOperationException; +067import org.apache.hadoop.hbase.MasterNotRunningException; +068import org.apache.hadoop.hbase.MetaTableAccessor; +069import org.apache.hadoop.hbase.NamespaceDescriptor; +070import org.apache.hadoop.hbase.PleaseHoldException; +071import org.apache.hadoop.hbase.ReplicationPeerNotFoundException; +072import org.apache.hadoop.hbase.ServerLoad; +073import org.apache.hadoop.hbase.ServerMetricsBuilder; +074import org.apache.hadoop.hbase.ServerName; +075import org.apache.hadoop.hbase.TableDescriptors; +076import org.apache.hadoop.hbase.TableName; +077import org.apache.hadoop.hbase.TableNotDisabledException; +078import org.apache.hadoop.hbase.TableNotFoundException; +079import org.apache.hadoop.hbase.UnknownRegionException; +080import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +081import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +082import org.apache.hadoop.hbase.client.MasterSwitchType; +083import org.apache.hadoop.hbase.client.RegionInfo; +084import org.apache.hadoop.hbase.client.Result; +085import org.apache.hadoop.hbase.client.TableDescriptor; +086import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +087import org.apache.hadoop.hbase.client.TableState; +088import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +089import org.apache.hadoop.hbase.exceptions.DeserializationException; +090import org.apache.hadoop.hbase.exceptions.MergeRegionException; +091import org.apache.hadoop.hbase.executor.ExecutorType; +092import org.apache.hadoop.hbase.favored.FavoredNodesManager; +093import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; +094import org.apache.hadoop.hbase.http.InfoServer; +095import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +096import org.apache.hadoop.hbase.ipc.RpcServer; +097import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +098import org.apache.hadoop.hbase.log.HBaseMarkers; +099import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; +100import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +101import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure; +102import org.apache.hadoop.hbase.master.assignment.RegionStates; +103import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode; +104import org.apache.hadoop.hbase.master.balancer.BalancerChore; +105import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; +106import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; +107import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +108import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +109import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +110import org.apache.hadoop.hbase.master.cleaner.ReplicationMetaCleaner; +111import org.apache.hadoop.hbase.master.locking.LockManager; +112import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; +113import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +114import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; +115import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; +116import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; +117import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; +118import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; +119import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; +120import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; +121import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +122import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +123import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; +124import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; +125import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; +126import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; +127import org.apache.hadoop.hbase.master.procedure.RecoverMetaProcedure; +128import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; +129import org.apache.hadoop.hbase.master.replication.AddPeerProcedure; +130import org.apache.hadoop.hbase.master.replication.D