hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/3] hbase git commit: HBASE-12467 Master joins cluster but never completes initialization
Date Wed, 10 Dec 2014 00:53:36 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 cb88d78f0 -> b9e6595ad
  refs/heads/branch-1 86c856320 -> 4a1d56a4a
  refs/heads/master 1be63539f -> cc8bdcb49


HBASE-12467 Master joins cluster but never completes initialization

Signed-off-by: Andrew Purtell <apurtell@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cc8bdcb4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cc8bdcb4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cc8bdcb4

Branch: refs/heads/master
Commit: cc8bdcb498e1aba022be9ff0c8fd533a1ea5f83c
Parents: 1be6353
Author: Nick Dimiduk <ndimiduk@apache.org>
Authored: Mon Nov 17 09:23:09 2014 +0100
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Tue Dec 9 16:32:11 2014 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/master/HMaster.java | 60 ++++++++++++++++++++
 1 file changed, 60 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/cc8bdcb4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9e7a679..b617519 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.PrintWriter;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.InetAddress;
@@ -35,6 +36,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 
@@ -119,6 +121,7 @@ import org.apache.hadoop.hbase.util.CompressionTest;
 import org.apache.hadoop.hbase.util.EncryptionTest;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
@@ -162,6 +165,59 @@ import com.google.protobuf.Service;
 public class HMaster extends HRegionServer implements MasterServices, Server {
   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
 
+  /**
+   * Protection against zombie master. Started once Master accepts active responsibility
and
+   * starts taking over responsibilities. Allows a finite time window before giving up ownership.
+   */
+  private static class InitializationMonitor extends HasThread {
+    /** The amount of time in milliseconds to sleep before checking initialization status.
*/
+    public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
+    public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
+
+    /**
+     * When timeout expired and initialization has not complete, call {@link System#exit(int)}
when
+     * true, do nothing otherwise.
+     */
+    public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
+    public static final boolean HALT_DEFAULT = false;
+
+    private final HMaster master;
+    private final long timeout;
+    private final boolean haltOnTimeout;
+
+    /** Creates a Thread that monitors the {@link #isInitialized()} state. */
+    InitializationMonitor(HMaster master) {
+      super("MasterInitializationMonitor");
+      this.master = master;
+      this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
+      this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
+      this.setDaemon(true);
+    }
+
+    @Override
+    public void run() {
+      try {
+        while (!master.isStopped() && master.isActiveMaster()) {
+          Thread.sleep(timeout);
+          if (master.isInitialized()) {
+            LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
+          } else {
+            LOG.error("Master failed to complete initialization after " + timeout + "ms.
Please"
+                + " consider submitting a bug report including a thread dump of this process.");
+            if (haltOnTimeout) {
+              LOG.error("Zombie Master exiting. Thread dump to stdout");
+              org.apache.hadoop.util.ReflectionUtils.printThreadInfo(
+                  new PrintWriter(System.out), "Zombie HMaster");
+              System.exit(-1);
+            }
+          }
+        }
+      } catch (InterruptedException ie) {
+        LOG.trace("InitMonitor thread interrupted. Existing.");
+      }
+    }
+  }
+
   // MASTER is name of the webapp and the attribute name used stuffing this
   //instance into web context.
   public static final String MASTER = "master";
@@ -521,6 +577,8 @@ public class HMaster extends HRegionServer implements MasterServices,
Server {
       throws IOException, InterruptedException, KeeperException, CoordinatedStateException
{
 
     isActiveMaster = true;
+    Thread zombieDetector = new Thread(new InitializationMonitor(this));
+    zombieDetector.start();
 
     /*
      * We are active master now... go initialize components we need to run.
@@ -686,6 +744,8 @@ public class HMaster extends HRegionServer implements MasterServices,
Server {
         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
       }
     }
+
+    zombieDetector.interrupt();
   }
 
   /**


Mime
View raw message