hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1584346 - in /hbase/trunk: hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ hbase-common/src/main/java/org/apache/hadoop/hbase/ hbase-common/src/main/resources/ hbase-server/src/main/java/org/apache/hadoop/hbase/master/ hbase-...
Date Thu, 03 Apr 2014 19:39:56 GMT
Author: stack
Date: Thu Apr  3 19:39:55 2014
New Revision: 1584346

URL: http://svn.apache.org/r1584346
Log:
HBASE-10856 Enable distributed log replay as default

Modified:
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
Thu Apr  3 19:39:55 2014
@@ -505,7 +505,7 @@ public class RecoverableZooKeeper {
                   Bytes.toStringBinary(data));
               throw e;
             }
-            LOG.info("Node " + path + " already exists");
+            LOG.debug("Node " + path + " already exists");
             throw e;
 
           case CONNECTIONLOSS:

Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Apr
 3 19:39:55 2014
@@ -794,7 +794,10 @@ public final class HConstants {
 
   /** Conf key that enables unflushed WAL edits directly being replayed to region servers
*/
   public static final String DISTRIBUTED_LOG_REPLAY_KEY = "hbase.master.distributed.log.replay";
-  public static final boolean DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG = false;
+  /**
+   * Default 'distributed log replay' as true since hbase 0.99.0
+   */
+  public static final boolean DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG = true;
   public static final String DISALLOW_WRITES_IN_RECOVERING =
       "hbase.regionserver.disallow.writes.when.recovering";
   public static final boolean DEFAULT_DISALLOW_WRITES_IN_RECOVERING_CONFIG = false;

Modified: hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/resources/hbase-default.xml?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/hbase-common/src/main/resources/hbase-default.xml Thu Apr  3 19:39:55 2014
@@ -220,6 +220,17 @@ possible configurations would overwhelm 
     <description>The HLog file writer implementation.</description>
   </property>
   <property>
+    <name>hbase.master.distributed.log.replay</name>
+    <value>true</value>
+    <description>Enable 'distributed log replay' as default engine splitting
+    WAL files on server crash.  This default is new in hbase 1.0.  To fall
+    back to the old mode 'distributed log splitter', set the value to
+    'false'.  'Disributed log replay' improves MTTR because it does not
+    write intermediate files.  'DLR' required that 'hfile.format.version'
+    be set to version 3 or higher. 
+    </description>
+  </property>
+  <property>
     <name>hbase.regionserver.global.memstore.size</name>
     <value>0.4</value>
     <description>Maximum size of all memstores in a region server before new

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
Thu Apr  3 19:39:55 2014
@@ -115,7 +115,7 @@ public class DeadServer {
   }
 
   public synchronized void finish(ServerName sn) {
-    LOG.info("Finished processing " + sn);
+    LOG.debug("Finished processing " + sn);
     this.numProcessing--;
   }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
Thu Apr  3 19:39:55 2014
@@ -628,7 +628,7 @@ public class SplitLogManager extends Zoo
     try {
       if (regions == null) {
         // remove all children under /home/recovering-regions
-        LOG.info("Garbage collecting all recovering regions.");
+        LOG.debug("Garbage collecting all recovering region znodes");
         ZKUtil.deleteChildrenRecursively(watcher, watcher.recoveringRegionsZNode);
       } else {
         for (String curRegion : regions) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
Thu Apr  3 19:39:55 2014
@@ -88,7 +88,7 @@ public class CreateTableHandler extends 
     int timeout = conf.getInt("hbase.client.catalog.timeout", 10000);
     // Need hbase:meta availability to create a table
     try {
-      if(catalogTracker.waitForMeta(timeout) == null) {
+      if (catalogTracker.waitForMeta(timeout) == null) {
         throw new NotAllMetaRegionsOnlineException();
       }
     } catch (InterruptedException e) {
@@ -114,7 +114,9 @@ public class CreateTableHandler extends 
       // Use enabling state to tell if there is already a request for the same
       // table in progress. This will introduce a new zookeeper call. Given
       // createTable isn't a frequent operation, that should be ok.
-      //TODO: now that we have table locks, re-evaluate above
+      // TODO: now that we have table locks, re-evaluate above -- table locks are not enough.
+      // We could have cleared the hbase.rootdir and not zk.  How can we detect this case?
+      // Having to clean zk AND hdfs is awkward.
       try {
         if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName)) {
           throw new TableExistsException(tableName);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java
Thu Apr  3 19:39:55 2014
@@ -83,6 +83,6 @@ public class LogReplayHandler extends Ev
       this.deadServers.finish(serverName);
     }
     // logReplay is the last step of SSH so log a line to indicate that
-    LOG.info("Finished processing of shutdown of " + serverName);
+    LOG.info("Finished processing shutdown of " + serverName);
   }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1584346&r1=1584345&r2=1584346&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Thu Apr  3 19:39:55 2014
@@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.client.Mu
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.SplitLogManager;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -1972,7 +1973,13 @@ public class HLogSplitter {
    * @return true when distributed log replay is turned on
    */
   public static boolean isDistributedLogReplay(Configuration conf) {
-    return conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
+    boolean dlr = conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
       HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
+    int version = conf.getInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Distributed log replay=" + dlr + ", " + HFile.FORMAT_VERSION_KEY + "=" +
version);
+    }
+    // For distributed log replay, hfile version must be 3 at least; we need tag support.
+    return dlr && (version >= 3);
   }
 }



Mime
View raw message