Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0B95F18321 for ; Sun, 22 Nov 2015 22:40:14 +0000 (UTC) Received: (qmail 89662 invoked by uid 500); 22 Nov 2015 22:40:12 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 89553 invoked by uid 500); 22 Nov 2015 22:40:12 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 88954 invoked by uid 99); 22 Nov 2015 22:40:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 22 Nov 2015 22:40:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 273A2DFFD9; Sun, 22 Nov 2015 22:40:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Sun, 22 Nov 2015 22:40:21 -0000 Message-Id: <86ccd3fdc2fe40be9346f5cb52f00a91@git.apache.org> In-Reply-To: <9992cfe48d434570abd843a624c1b57d@git.apache.org> References: <9992cfe48d434570abd843a624c1b57d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [11/51] [partial] hbase git commit: Published site at 86be690b0723e814a655ad0ae8a6577d7111c1f2. http://git-wip-us.apache.org/repos/asf/hbase/blob/155d5cd0/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html index 9594ab6..e8d84bf 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html @@ -99,25 +99,25 @@ 091import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; 092import org.apache.hadoop.hbase.master.cleaner.LogCleaner; 093import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler; -094import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure; -095import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure; -096import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; -097import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure; -098import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure; -099import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; -100import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; -101import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; -102import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; -103import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -104import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; -105import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure; -106import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; -107import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; -108import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; -109import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; -110import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; -111import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; -112import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; +094import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; +095import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; +096import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; +097import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure; +098import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure; +099import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; +100import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure; +101import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure; +102import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; +103import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; +104import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; +105import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +106import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +107import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; +108import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure; +109import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; +110import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; +111import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; +112import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; 113import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; 114import org.apache.hadoop.hbase.mob.MobConstants; 115import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; @@ -132,2665 +132,2745 @@ 124import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; 125import org.apache.hadoop.hbase.quotas.MasterQuotaManager; 126import org.apache.hadoop.hbase.quotas.RegionStateListener; -127import org.apache.hadoop.hbase.regionserver.HRegionServer; -128import org.apache.hadoop.hbase.regionserver.RSRpcServices; -129import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; -130import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; -131import org.apache.hadoop.hbase.replication.regionserver.Replication; -132import org.apache.hadoop.hbase.security.UserProvider; -133import org.apache.hadoop.hbase.util.Addressing; -134import org.apache.hadoop.hbase.util.Bytes; -135import org.apache.hadoop.hbase.util.CompressionTest; -136import org.apache.hadoop.hbase.util.EncryptionTest; -137import org.apache.hadoop.hbase.util.FSUtils; -138import org.apache.hadoop.hbase.util.HFileArchiveUtil; -139import org.apache.hadoop.hbase.util.HasThread; -140import org.apache.hadoop.hbase.util.IdLock; -141import org.apache.hadoop.hbase.util.ModifyRegionUtils; -142import org.apache.hadoop.hbase.util.Pair; -143import org.apache.hadoop.hbase.util.Threads; -144import org.apache.hadoop.hbase.util.VersionInfo; -145import org.apache.hadoop.hbase.util.ZKDataMigrator; -146import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; -147import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; -148import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -149import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -150import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; -151import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; -152import org.apache.hadoop.hbase.zookeeper.ZKClusterId; -153import org.apache.hadoop.hbase.zookeeper.ZKUtil; -154import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -155import org.apache.zookeeper.KeeperException; -156import org.mortbay.jetty.Connector; -157import org.mortbay.jetty.nio.SelectChannelConnector; -158import org.mortbay.jetty.servlet.Context; -159 -160import com.google.common.annotations.VisibleForTesting; -161import com.google.common.collect.Maps; -162import com.google.protobuf.Descriptors; -163import com.google.protobuf.Service; -164 -165/** -166 * HMaster is the "master server" for HBase. An HBase cluster has one active -167 * master. If many masters are started, all compete. Whichever wins goes on to -168 * run the cluster. All others park themselves in their constructor until -169 * master or cluster shutdown or until the active master loses its lease in -170 * zookeeper. Thereafter, all running master jostle to take over master role. -171 * -172 * The Master can be asked shutdown the cluster. See {@link #shutdown()}. In -173 * this case it will tell all regionservers to go down and then wait on them -174 * all reporting in that they are down. This master will then shut itself down. +127import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; +128import org.apache.hadoop.hbase.regionserver.HRegionServer; +129import org.apache.hadoop.hbase.regionserver.HStore; +130import org.apache.hadoop.hbase.regionserver.RSRpcServices; +131import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +132import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; +133import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; +134import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; +135import org.apache.hadoop.hbase.replication.regionserver.Replication; +136import org.apache.hadoop.hbase.security.UserProvider; +137import org.apache.hadoop.hbase.util.Addressing; +138import org.apache.hadoop.hbase.util.Bytes; +139import org.apache.hadoop.hbase.util.CompressionTest; +140import org.apache.hadoop.hbase.util.EncryptionTest; +141import org.apache.hadoop.hbase.util.FSUtils; +142import org.apache.hadoop.hbase.util.HFileArchiveUtil; +143import org.apache.hadoop.hbase.util.HasThread; +144import org.apache.hadoop.hbase.util.IdLock; +145import org.apache.hadoop.hbase.util.ModifyRegionUtils; +146import org.apache.hadoop.hbase.util.Pair; +147import org.apache.hadoop.hbase.util.Threads; +148import org.apache.hadoop.hbase.util.VersionInfo; +149import org.apache.hadoop.hbase.util.ZKDataMigrator; +150import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; +151import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; +152import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; +153import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +154import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; +155import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; +156import org.apache.hadoop.hbase.zookeeper.ZKClusterId; +157import org.apache.hadoop.hbase.zookeeper.ZKUtil; +158import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +159import org.apache.zookeeper.KeeperException; +160import org.mortbay.jetty.Connector; +161import org.mortbay.jetty.nio.SelectChannelConnector; +162import org.mortbay.jetty.servlet.Context; +163 +164import com.google.common.annotations.VisibleForTesting; +165import com.google.common.collect.Maps; +166import com.google.protobuf.Descriptors; +167import com.google.protobuf.Service; +168 +169/** +170 * HMaster is the "master server" for HBase. An HBase cluster has one active +171 * master. If many masters are started, all compete. Whichever wins goes on to +172 * run the cluster. All others park themselves in their constructor until +173 * master or cluster shutdown or until the active master loses its lease in +174 * zookeeper. Thereafter, all running master jostle to take over master role. 175 * -176 * You can also shutdown just this master. Call {@link #stopMaster()}. -177 * -178 * @see org.apache.zookeeper.Watcher -179 */ -180@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -181@SuppressWarnings("deprecation") -182public class HMaster extends HRegionServer implements MasterServices, Server { -183 private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); -184 -185 /** -186 * Protection against zombie master. Started once Master accepts active responsibility and -187 * starts taking over responsibilities. Allows a finite time window before giving up ownership. -188 */ -189 private static class InitializationMonitor extends HasThread { -190 /** The amount of time in milliseconds to sleep before checking initialization status. */ -191 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout"; -192 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES); -193 -194 /** -195 * When timeout expired and initialization has not complete, call {@link System#exit(int)} when -196 * true, do nothing otherwise. -197 */ -198 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout"; -199 public static final boolean HALT_DEFAULT = false; -200 -201 private final HMaster master; -202 private final long timeout; -203 private final boolean haltOnTimeout; +176 * The Master can be asked shutdown the cluster. See {@link #shutdown()}. In +177 * this case it will tell all regionservers to go down and then wait on them +178 * all reporting in that they are down. This master will then shut itself down. +179 * +180 * You can also shutdown just this master. Call {@link #stopMaster()}. +181 * +182 * @see org.apache.zookeeper.Watcher +183 */ +184@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +185@SuppressWarnings("deprecation") +186public class HMaster extends HRegionServer implements MasterServices, Server { +187 private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); +188 +189 /** +190 * Protection against zombie master. Started once Master accepts active responsibility and +191 * starts taking over responsibilities. Allows a finite time window before giving up ownership. +192 */ +193 private static class InitializationMonitor extends HasThread { +194 /** The amount of time in milliseconds to sleep before checking initialization status. */ +195 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout"; +196 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES); +197 +198 /** +199 * When timeout expired and initialization has not complete, call {@link System#exit(int)} when +200 * true, do nothing otherwise. +201 */ +202 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout"; +203 public static final boolean HALT_DEFAULT = false; 204 -205 /** Creates a Thread that monitors the {@link #isInitialized()} state. */ -206 InitializationMonitor(HMaster master) { -207 super("MasterInitializationMonitor"); -208 this.master = master; -209 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT); -210 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT); -211 this.setDaemon(true); -212 } -213 -214 @Override -215 public void run() { -216 try { -217 while (!master.isStopped() && master.isActiveMaster()) { -218 Thread.sleep(timeout); -219 if (master.isInitialized()) { -220 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting."); -221 } else { -222 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please" -223 + " consider submitting a bug report including a thread dump of this process."); -224 if (haltOnTimeout) { -225 LOG.error("Zombie Master exiting. Thread dump to stdout"); -226 Threads.printThreadInfo(System.out, "Zombie HMaster"); -227 System.exit(-1); -228 } -229 } -230 } -231 } catch (InterruptedException ie) { -232 LOG.trace("InitMonitor thread interrupted. Existing."); -233 } -234 } -235 } -236 -237 // MASTER is name of the webapp and the attribute name used stuffing this -238 //instance into web context. -239 public static final String MASTER = "master"; +205 private final HMaster master; +206 private final long timeout; +207 private final boolean haltOnTimeout; +208 +209 /** Creates a Thread that monitors the {@link #isInitialized()} state. */ +210 InitializationMonitor(HMaster master) { +211 super("MasterInitializationMonitor"); +212 this.master = master; +213 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT); +214 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT); +215 this.setDaemon(true); +216 } +217 +218 @Override +219 public void run() { +220 try { +221 while (!master.isStopped() && master.isActiveMaster()) { +222 Thread.sleep(timeout); +223 if (master.isInitialized()) { +224 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting."); +225 } else { +226 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please" +227 + " consider submitting a bug report including a thread dump of this process."); +228 if (haltOnTimeout) { +229 LOG.error("Zombie Master exiting. Thread dump to stdout"); +230 Threads.printThreadInfo(System.out, "Zombie HMaster"); +231 System.exit(-1); +232 } +233 } +234 } +235 } catch (InterruptedException ie) { +236 LOG.trace("InitMonitor thread interrupted. Existing."); +237 } +238 } +239 } 240 -241 // Manager and zk listener for master election -242 private final ActiveMasterManager activeMasterManager; -243 // Region server tracker -244 RegionServerTracker regionServerTracker; -245 // Draining region server tracker -246 private DrainingServerTracker drainingServerTracker; -247 // Tracker for load balancer state -248 LoadBalancerTracker loadBalancerTracker; -249 -250 // Tracker for region normalizer state -251 private RegionNormalizerTracker regionNormalizerTracker; -252 -253 /** Namespace stuff */ -254 private TableNamespaceManager tableNamespaceManager; -255 -256 // Metrics for the HMaster -257 final MetricsMaster metricsMaster; -258 // file system manager for the master FS operations -259 private MasterFileSystem fileSystemManager; -260 -261 // server manager to deal with region server info -262 volatile ServerManager serverManager; -263 -264 // manager of assignment nodes in zookeeper -265 AssignmentManager assignmentManager; -266 -267 // buffer for "fatal error" notices from region servers -268 // in the cluster. This is only used for assisting -269 // operations/debugging. -270 MemoryBoundedLogMessageBuffer rsFatals; -271 -272 // flag set after we become the active master (used for testing) -273 private volatile boolean isActiveMaster = false; -274 -275 // flag set after we complete initialization once active, -276 // it is not private since it's used in unit tests -277 volatile boolean initialized = false; +241 // MASTER is name of the webapp and the attribute name used stuffing this +242 //instance into web context. +243 public static final String MASTER = "master"; +244 +245 // Manager and zk listener for master election +246 private final ActiveMasterManager activeMasterManager; +247 // Region server tracker +248 RegionServerTracker regionServerTracker; +249 // Draining region server tracker +250 private DrainingServerTracker drainingServerTracker; +251 // Tracker for load balancer state +252 LoadBalancerTracker loadBalancerTracker; +253 +254 // Tracker for region normalizer state +255 private RegionNormalizerTracker regionNormalizerTracker; +256 +257 /** Namespace stuff */ +258 private TableNamespaceManager tableNamespaceManager; +259 +260 // Metrics for the HMaster +261 final MetricsMaster metricsMaster; +262 // file system manager for the master FS operations +263 private MasterFileSystem fileSystemManager; +264 +265 // server manager to deal with region server info +266 volatile ServerManager serverManager; +267 +268 // manager of assignment nodes in zookeeper +269 AssignmentManager assignmentManager; +270 +271 // buffer for "fatal error" notices from region servers +272 // in the cluster. This is only used for assisting +273 // operations/debugging. +274 MemoryBoundedLogMessageBuffer rsFatals; +275 +276 // flag set after we become the active master (used for testing) +277 private volatile boolean isActiveMaster = false; 278 -279 // flag set after master services are started, -280 // initialization may have not completed yet. -281 volatile boolean serviceStarted = false; +279 // flag set after we complete initialization once active, +280 // it is not private since it's used in unit tests +281 volatile boolean initialized = false; 282 -283 // flag set after we complete assignMeta. -284 private volatile boolean serverCrashProcessingEnabled = false; -285 -286 LoadBalancer balancer; -287 private RegionNormalizer normalizer; -288 private BalancerChore balancerChore; -289 private RegionNormalizerChore normalizerChore; -290 private ClusterStatusChore clusterStatusChore; -291 private ClusterStatusPublisher clusterStatusPublisherChore = null; -292 -293 CatalogJanitor catalogJanitorChore; -294 private LogCleaner logCleaner; -295 private HFileCleaner hfileCleaner; -296 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; -297 private MobCompactionChore mobCompactChore; -298 private MasterMobCompactionThread mobCompactThread; -299 // used to synchronize the mobCompactionStates -300 private final IdLock mobCompactionLock = new IdLock(); -301 // save the information of mob compactions in tables. -302 // the key is table name, the value is the number of compactions in that table. -303 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap(); -304 -305 MasterCoprocessorHost cpHost; -306 -307 private final boolean preLoadTableDescriptors; +283 // flag set after master services are started, +284 // initialization may have not completed yet. +285 volatile boolean serviceStarted = false; +286 +287 // flag set after we complete assignMeta. +288 private volatile boolean serverCrashProcessingEnabled = false; +289 +290 LoadBalancer balancer; +291 private RegionNormalizer normalizer; +292 private BalancerChore balancerChore; +293 private RegionNormalizerChore normalizerChore; +294 private ClusterStatusChore clusterStatusChore; +295 private ClusterStatusPublisher clusterStatusPublisherChore = null; +296 +297 CatalogJanitor catalogJanitorChore; +298 private LogCleaner logCleaner; +299 private HFileCleaner hfileCleaner; +300 private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; +301 private MobCompactionChore mobCompactChore; +302 private MasterMobCompactionThread mobCompactThread; +303 // used to synchronize the mobCompactionStates +304 private final IdLock mobCompactionLock = new IdLock(); +305 // save the information of mob compactions in tables. +306 // the key is table name, the value is the number of compactions in that table. +307 private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap(); 308 -309 // Time stamps for when a hmaster became active -310 private long masterActiveTime; -311 -312 //should we check the compression codec type at master side, default true, HBASE-6370 -313 private final boolean masterCheckCompression; -314 -315 //should we check encryption settings at master side, default true -316 private final boolean masterCheckEncryption; -317 -318 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); -319 -320 // monitor for snapshot of hbase tables -321 SnapshotManager snapshotManager; -322 // monitor for distributed procedures -323 MasterProcedureManagerHost mpmHost; -324 -325 // it is assigned after 'initialized' guard set to true, so should be volatile -326 private volatile MasterQuotaManager quotaManager; -327 -328 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; -329 private WALProcedureStore procedureStore; -330 -331 // handle table states -332 private TableStateManager tableStateManager; -333 -334 /** flag used in test cases in order to simulate RS failures during master initialization */ -335 private volatile boolean initializationBeforeMetaAssignment = false; -336 -337 /** jetty server for master to redirect requests to regionserver infoServer */ -338 private org.mortbay.jetty.Server masterJettyServer; -339 -340 public static class RedirectServlet extends HttpServlet { -341 private static final long serialVersionUID = 2894774810058302472L; -342 private static int regionServerInfoPort; +309 MasterCoprocessorHost cpHost; +310 +311 private final boolean preLoadTableDescriptors; +312 +313 // Time stamps for when a hmaster became active +314 private long masterActiveTime; +315 +316 //should we check the compression codec type at master side, default true, HBASE-6370 +317 private final boolean masterCheckCompression; +318 +319 //should we check encryption settings at master side, default true +320 private final boolean masterCheckEncryption; +321 +322 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); +323 +324 // monitor for snapshot of hbase tables +325 SnapshotManager snapshotManager; +326 // monitor for distributed procedures +327 MasterProcedureManagerHost mpmHost; +328 +329 // it is assigned after 'initialized' guard set to true, so should be volatile +330 private volatile MasterQuotaManager quotaManager; +331 +332 private ProcedureExecutor<MasterProcedureEnv> procedureExecutor; +333 private WALProcedureStore procedureStore; +334 +335 // handle table states +336 private TableStateManager tableStateManager; +337 +338 /** flag used in test cases in order to simulate RS failures during master initialization */ +339 private volatile boolean initializationBeforeMetaAssignment = false; +340 +341 /** jetty server for master to redirect requests to regionserver infoServer */ +342 private org.mortbay.jetty.Server masterJettyServer; 343 -344 @Override -345 public void doGet(HttpServletRequest request, -346 HttpServletResponse response) throws ServletException, IOException { -347 String redirectUrl = request.getScheme() + "://" -348 + request.getServerName() + ":" + regionServerInfoPort -349 + request.getRequestURI(); -350 response.sendRedirect(redirectUrl); -351 } -352 } -353 -354 /** -355 * Initializes the HMaster. The steps are as follows: -356 * -357 * <ol> -358 * <li>Initialize the local HRegionServer -359 * <li>Start the ActiveMasterManager. -360 * </ol> -361 * -362 * Remaining steps of initialization occur in -363 * #finishActiveMasterInitialization(MonitoredTask) after -364 * the master becomes the active one. -365 * -366 * @throws KeeperException -367 * @throws IOException -368 */ -369 public HMaster(final Configuration conf, CoordinatedStateManager csm) -370 throws IOException, KeeperException { -371 super(conf, csm); -372 this.rsFatals = new MemoryBoundedLogMessageBuffer( -373 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024)); -374 -375 LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) + -376 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false)); -377 -378 // Disable usage of meta replicas in the master -379 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); -380 -381 Replication.decorateMasterConfiguration(this.conf); -382 -383 // Hack! Maps DFSClient => Master for logs. HDFS made this -384 // config param for task trackers, but we can piggyback off of it. -385 if (this.conf.get("mapreduce.task.attempt.id") == null) { -386 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString()); -387 } -388 -389 // should we check the compression codec type at master side, default true, HBASE-6370 -390 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true); -391 -392 // should we check encryption settings at master side, default true -393 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true); -394 -395 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); -396 -397 // preload table descriptor at startup -398 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); -399 -400 // Do we publish the status? -401 -402 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, -403 HConstants.STATUS_PUBLISHED_DEFAULT); -404 Class<? extends ClusterStatusPublisher.Publisher> publisherClass = -405 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS, -406 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS, -407 ClusterStatusPublisher.Publisher.class); -408 -409 if (shouldPublish) { -410 if (publisherClass == null) { -411 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " + -412 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS + -413 " is not set - not publishing status"); -414 } else { -415 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass); -416 getChoreService().scheduleChore(clusterStatusPublisherChore); -417 } -418 } -419 -420 // Some unit tests don't need a cluster, so no zookeeper at all -421 if (!conf.getBoolean("hbase.testing.nocluster", false)) { -422 activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); -423 int infoPort = putUpJettyServer(); -424 startActiveMasterManager(infoPort); -425 } else { -426 activeMasterManager = null; -427 } -428 } -429 -430 // return the actual infoPort, -1 means disable info server. -431 private int putUpJettyServer() throws IOException { -432 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { -433 return -1; -434 } -435 int infoPort = conf.getInt("hbase.master.info.port.orig", -436 HConstants.DEFAULT_MASTER_INFOPORT); -437 // -1 is for disabling info server, so no redirecting -438 if (infoPort < 0 || infoServer == null) { -439 return -1; -440 } -441 String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); -442 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) { -443 String msg = -444 "Failed to start redirecting jetty server. Address " + addr -445 + " does not belong to this host. Correct configuration parameter: " -446 + "hbase.master.info.bindAddress"; -447 LOG.error(msg); -448 throw new IOException(msg); -449 } -450 -451 RedirectServlet.regionServerInfoPort = infoServer.getPort(); -452 if(RedirectServlet.regionServerInfoPort == infoPort) { -453 return infoPort; -454 } -455 masterJettyServer = new org.mortbay.jetty.Server(); -456 Connector connector = new SelectChannelConnector(); -457 connector.setHost(addr); -458 connector.setPort(infoPort); -459 masterJettyServer.addConnector(connector); -460 masterJettyServer.setStopAtShutdown(true); -461 Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS); -462 context.addServlet(RedirectServlet.class, "/*"); -463 try { -464 masterJettyServer.start(); -465 } catch (Exception e) { -466 throw new IOException("Failed to start redirecting jetty server", e); -467 } -468 return connector.getLocalPort(); -469 } -470 -471 @Override -472 protected TableDescriptors getFsTableDescriptors() throws IOException { -473 return super.getFsTableDescriptors(); -474 } -475 -476 /** -477 * For compatibility, if failed with regionserver credentials, try the master one -478 */ -479 @Override -480 protected void login(UserProvider user, String host) throws IOException { -481 try { -482 super.login(user, host); -483 } catch (IOException ie) { -484 user.login("hbase.master.keytab.file", -485 "hbase.master.kerberos.principal", host); -486 } -487 } -488 -489 /** -490 * If configured to put regions on active master, -491 * wait till a backup master becomes active. -492 * Otherwise, loop till the server is stopped or aborted. -493 */ -494 @Override -495 protected void waitForMasterActive(){ -496 boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf); -497 while (!(tablesOnMaster && isActiveMaster) -498 && !isStopped() && !isAborted()) { -499 sleeper.sleep(); -500 } -501 } -502 -503 @VisibleForTesting -504 public MasterRpcServices getMasterRpcServices() { -505 return (MasterRpcServices)rpcServices; -506 } -507 -508 public boolean balanceSwitch(final boolean b) throws IOException { -509 return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC); +344 public static class RedirectServlet extends HttpServlet { +345 private static final long serialVersionUID = 2894774810058302472L; +346 private static int regionServerInfoPort; +347 +348 @Override +349 public void doGet(HttpServletRequest request, +350 HttpServletResponse response) throws ServletException, IOException { +351 String redirectUrl = request.getScheme() + "://" +352 + request.getServerName() + ":" + regionServerInfoPort +353 + request.getRequestURI(); +354 response.sendRedirect(redirectUrl); +355 } +356 } +357 +358 /** +359 * Initializes the HMaster. The steps are as follows: +360 * +361 * <ol> +362 * <li>Initialize the local HRegionServer +363 * <li>Start the ActiveMasterManager. +364 * </ol> +365 * +366 * Remaining steps of initialization occur in +367 * #finishActiveMasterInitialization(MonitoredTask) after +368 * the master becomes the active one. +369 * +370 * @throws KeeperException +371 * @throws IOException +372 */ +373 public HMaster(final Configuration conf, CoordinatedStateManager csm) +374 throws IOException, KeeperException { +375 super(conf, csm); +376 this.rsFatals = new MemoryBoundedLogMessageBuffer( +377 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024)); +378 +379 LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) + +380 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false)); +381 +382 // Disable usage of meta replicas in the master +383 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); +384 +385 Replication.decorateMasterConfiguration(this.conf); +386 +387 // Hack! Maps DFSClient => Master for logs. HDFS made this +388 // config param for task trackers, but we can piggyback off of it. +389 if (this.conf.get("mapreduce.task.attempt.id") == null) { +390 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString()); +391 } +392 +393 // should we check the compression codec type at master side, default true, HBASE-6370 +394 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true); +395 +396 // should we check encryption settings at master side, default true +397 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true); +398 +399 this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); +400 +401 // preload table descriptor at startup +402 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); +403 +404 // Do we publish the status? +405 +406 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, +407 HConstants.STATUS_PUBLISHED_DEFAULT); +408 Class<? extends ClusterStatusPublisher.Publisher> publisherClass = +409 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS, +410 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS, +411 ClusterStatusPublisher.Publisher.class); +412 +413 if (shouldPublish) { +414 if (publisherClass == null) { +415 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " + +416 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS + +417 " is not set - not publishing status"); +418 } else { +419 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass); +420 getChoreService().scheduleChore(clusterStatusPublisherChore); +421 } +422 } +423 +424 // Some unit tests don't need a cluster, so no zookeeper at all +425 if (!conf.getBoolean("hbase.testing.nocluster", false)) { +426 activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); +427 int infoPort = putUpJettyServer(); +428 startActiveMasterManager(infoPort); +429 } else { +430 activeMasterManager = null; +431 } +432 } +433 +434 // return the actual infoPort, -1 means disable info server. +435 private int putUpJettyServer() throws IOException { +436 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { +437 return -1; +438 } +439 int infoPort = conf.getInt("hbase.master.info.port.orig", +440 HConstants.DEFAULT_MASTER_INFOPORT); +441 // -1 is for disabling info server, so no redirecting +442 if (infoPort < 0 || infoServer == null) { +443 return -1; +444 } +445 String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); +446 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) { +447 String msg = +448 "Failed to start redirecting jetty server. Address " + addr +449 + " does not belong to this host. Correct configuration parameter: " +450 + "hbase.master.info.bindAddress"; +451 LOG.error(msg); +452 throw new IOException(msg); +453 } +454 +455 RedirectServlet.regionServerInfoPort = infoServer.getPort(); +456 if(RedirectServlet.regionServerInfoPort == infoPort) { +457 return infoPort; +458 } +459 masterJettyServer = new org.mortbay.jetty.Server(); +460 Connector connector = new SelectChannelConnector(); +461 connector.setHost(addr); +462 connector.setPort(infoPort); +463 masterJettyServer.addConnector(connector); +464 masterJettyServer.setStopAtShutdown(true); +465 Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS); +466 context.addServlet(RedirectServlet.class, "/*"); +467 try { +468 masterJettyServer.start(); +469 } catch (Exception e) { +470 throw new IOException("Failed to start redirecting jetty server", e); +471 } +472 return connector.getLocalPort(); +473 } +474 +475 @Override +476 protected TableDescriptors getFsTableDescriptors() throws IOException { +477 return super.getFsTableDescriptors(); +478 } +479 +480 /** +481 * For compatibility, if failed with regionserver credentials, try the master one +482 */ +483 @Override +484 protected void login(UserProvider user, String host) throws IOException { +485 try { +486 super.login(user, host); +487 } catch (IOException ie) { +488 user.login("hbase.master.keytab.file", +489 "hbase.master.kerberos.principal", host); +490 } +491 } +492 +493 /** +494 * If configured to put regions on active master, +495 * wait till a backup master becomes active. +496 * Otherwise, loop till the server is stopped or aborted. +497 */ +498 @Override +499 protected void waitForMasterActive(){ +500 boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf); +501 while (!(tablesOnMaster && isActiveMaster) +502 && !isStopped() && !isAborted()) { +503 sleeper.sleep(); +504 } +505 } +506 +507 @VisibleForTesting +508 public MasterRpcServices getMasterRpcServices() { +509 return (MasterRpcServices)rpcServices; 510 } 511 -512 @Override -513 protected String getProcessName() { -514 return MASTER; -515 } -516 -517 @Override -518 protected boolean canCreateBaseZNode() { -519 return true; -520 } -521 -522 @Override -523 protected boolean canUpdateTableDescriptor() { -524 return true; -525 } -526 -527 @Override -528 protected RSRpcServices createRpcServices() throws IOException { -529 return new MasterRpcServices(this); -530 } -531 -532 @Override -533 protected void configureInfoServer() { -534 infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class); -535 infoServer.setAttribute(MASTER, this); -536 if (BaseLoadBalancer.tablesOnMaster(conf)) { -537 super.configureInfoServer(); -538 } -539 } -540 -541 @Override -542 protected Class<? extends HttpServlet> getDumpServlet() { -543 return MasterDumpServlet.class; -544 } -545 -546 /** -547 * Emit the HMaster metrics, such as region in transition metrics. -548 * Surrounding in a try block just to be sure metrics doesn't abort HMaster. -549 */ -550 @Override -551 protected void doMetrics() { -552 try { -553 if (assignmentManager != null) { -554 assignmentManager.updateRegionsInTransitionMetrics(); -555 } -556 } catch (Throwable e) { -557 LOG.error("Couldn't update metrics: " + e.getMessage()); -558 } -559 } -560 -561 MetricsMaster getMasterMetrics() { -562 return metricsMaster; +512 public boolean balanceSwitch(final boolean b) throws IOException { +513 return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC); +514 } +515 +516 @Override +517 protected String getProcessName() { +518 return MASTER; +519 } +520 +521 @Override