hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject svn commit: r1528227 [2/2] - in /hbase/trunk: hbase-client/src/main/java/org/apache/hadoop/hbase/ hbase-it/src/test/java/org/apache/hadoop/hbase/ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ hbase-it/src/test/java/org/apache/hadoop/hba...
Date Tue, 01 Oct 2013 22:02:02 GMT
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
Tue Oct  1 22:02:01 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -29,6 +30,7 @@ import java.util.TreeMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.RegionTransition;
 import org.apache.hadoop.hbase.Server;
@@ -78,14 +80,50 @@ public class RegionStates {
    */
   private final TreeMap<HRegionInfo, ServerName> regionAssignments;
 
+  /**
+   * Encoded region name to server assignment map for re-assignment
+   * purpose. Contains the server a given region is last known assigned
+   * to, which has not completed log splitting, so not assignable.
+   * If a region is currently assigned, this server info in this
+   * map should be the same as that in regionAssignments.
+   * However the info in regionAssignments is cleared when the region
+   * is offline while the info in lastAssignments is cleared when
+   * the region is closed or the server is dead and processed.
+   */
+  private final HashMap<String, ServerName> lastAssignments;
+
+  /**
+   * Map a host port pair string to the latest start code
+   * of a region server which is known to be dead. It is dead
+   * to us, but server manager may not know it yet.
+   */
+  private final HashMap<String, Long> deadServers;
+
+  /**
+   * Map a dead servers to the time when log split is done.
+   * Since log splitting is not ordered, we have to remember
+   * all processed instances. The map is cleaned up based
+   * on a configured time. By default, we assume a dead
+   * server should be done with log splitting in two hours.
+   */
+  private final HashMap<ServerName, Long> processedServers;
+  private long lastProcessedServerCleanTime;
+
   private final ServerManager serverManager;
   private final Server server;
 
+  // The maximum time to keep a log split info in region states map
+  static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime";
+  static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours
+
   RegionStates(final Server master, final ServerManager serverManager) {
     regionStates = new HashMap<String, RegionState>();
     regionsInTransition = new HashMap<String, RegionState>();
     serverHoldings = new HashMap<ServerName, Set<HRegionInfo>>();
     regionAssignments = new TreeMap<HRegionInfo, ServerName>();
+    lastAssignments = new HashMap<String, ServerName>();
+    processedServers = new HashMap<ServerName, Long>();
+    deadServers = new HashMap<String, Long>();
     this.serverManager = serverManager;
     this.server = master;
   }
@@ -132,27 +170,32 @@ public class RegionStates {
   }
 
   /**
-   * @return True if specified region assigned.
+   * @return True if specified region assigned, and not in transition.
    */
-  public synchronized boolean isRegionAssigned(final HRegionInfo hri) {
-    return regionAssignments.containsKey(hri);
+  public synchronized boolean isRegionOnline(final HRegionInfo hri) {
+    return !isRegionInTransition(hri) && regionAssignments.containsKey(hri);
   }
 
   /**
-   * @return True if specified region offline.
+   * @return True if specified region offline/closed, but not in transition.
+   * If the region is not in the map, it is offline to us too.
    */
   public synchronized boolean isRegionOffline(final HRegionInfo hri) {
-    return !isRegionInTransition(hri) && isRegionInState(hri, State.OFFLINE);
+    return getRegionState(hri) == null || (!isRegionInTransition(hri)
+      && isRegionInState(hri, State.OFFLINE, State.CLOSED));
   }
 
   /**
-   * @return True if specified region is in specified state
+   * @return True if specified region is in one of the specified states.
    */
   public synchronized boolean isRegionInState(
-      final HRegionInfo hri, final State state) {
+      final HRegionInfo hri, final State... states) {
     RegionState regionState = getRegionState(hri);
     State s = regionState != null ? regionState.getState() : null;
-    return s == state;
+    for (State state: states) {
+      if (s == state) return true;
+    }
+    return false;
   }
 
   /**
@@ -252,8 +295,8 @@ public class RegionStates {
     ServerName newServerName = serverName;
     if (serverName != null &&
         (state == State.CLOSED || state == State.OFFLINE)) {
-      LOG.warn("Closed " + hri.getShortNameToLog() + " still on "
-        + serverName + "? Ignored, reset it to null");
+      LOG.info(hri.getShortNameToLog() + " is " + state
+        + ", reset server info from " + serverName + " to null");
       newServerName = null;
     }
 
@@ -274,6 +317,20 @@ public class RegionStates {
       regionsInTransition.put(regionName, regionState);
     }
 
+    // For these states, region should be properly closed.
+    // There should be no log splitting issue.
+    if ((state == State.CLOSED || state == State.MERGED
+        || state == State.SPLIT) && lastAssignments.containsKey(regionName)) {
+      ServerName oldServerName = oldState == null ? null : oldState.getServerName();
+      ServerName last = lastAssignments.get(regionName);
+      if (last.equals(oldServerName)) {
+        lastAssignments.remove(regionName);
+      } else {
+        LOG.warn(regionName + " moved to " + state + " on "
+          + oldServerName + ", expected " + last);
+      }
+    }
+
     // notify the change
     this.notifyAll();
     return regionState;
@@ -286,6 +343,16 @@ public class RegionStates {
    */
   public synchronized void regionOnline(
       final HRegionInfo hri, final ServerName serverName) {
+    if (!serverManager.isServerOnline(serverName)) {
+      // This is possible if the region server dies before master gets a
+      // chance to handle ZK event in time. At this time, if the dead server
+      // is already processed by SSH, we should ignore this event.
+      // If not processed yet, ignore and let SSH deal with it.
+      LOG.warn("Ignored, " + hri.getEncodedName()
+        + " was opened on a dead server: " + serverName);
+      return;
+    }
+
     String regionName = hri.getEncodedName();
     RegionState oldState = regionStates.get(regionName);
     if (oldState == null) {
@@ -301,6 +368,7 @@ public class RegionStates {
     updateRegionState(hri, State.OPEN, serverName);
     regionsInTransition.remove(regionName);
 
+    lastAssignments.put(regionName, serverName);
     ServerName oldServerName = regionAssignments.put(hri, serverName);
     if (!serverName.equals(oldServerName)) {
       LOG.info("Onlined " + hri.getShortNameToLog() + " on " + serverName);
@@ -312,12 +380,53 @@ public class RegionStates {
       regions.add(hri);
       if (oldServerName != null) {
         LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
-        serverHoldings.get(oldServerName).remove(hri);
+        Set<HRegionInfo> oldRegions = serverHoldings.get(oldServerName);
+        oldRegions.remove(hri);
+        if (oldRegions.isEmpty()) {
+          serverHoldings.remove(oldServerName);
+        }
       }
     }
   }
 
   /**
+   * A dead server's hlogs have been split so that all the regions
+   * used to be open on it can be safely assigned now. Mark them assignable.
+   */
+  public synchronized void logSplit(final ServerName serverName) {
+    for (Iterator<Map.Entry<String, ServerName>> it
+        = lastAssignments.entrySet().iterator(); it.hasNext();) {
+      Map.Entry<String, ServerName> e = it.next();
+      if (e.getValue().equals(serverName)) {
+        it.remove();
+      }
+    }
+    long now = System.currentTimeMillis();
+    processedServers.put(serverName, Long.valueOf(now));
+    Configuration conf = server.getConfiguration();
+    long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME);
+    // Doesn't have to be very accurate about the clean up time
+    if (now > lastProcessedServerCleanTime + obsoleteTime) {
+      lastProcessedServerCleanTime = now;
+      long cutoff = now - obsoleteTime;
+      for (Iterator<Map.Entry<ServerName, Long>> it
+          = processedServers.entrySet().iterator(); it.hasNext();) {
+        Map.Entry<ServerName, Long> e = it.next();
+        if (e.getValue().longValue() < cutoff) {
+          it.remove();
+        }
+      }
+    }
+  }
+
+  /**
+   * Log split is done for a given region, so it is assignable now.
+   */
+  public synchronized void logSplit(final HRegionInfo region) {
+    lastAssignments.remove(region.getEncodedName());
+  }
+
+  /**
    * A region is offline, won't be in transition any more.
    */
   public void regionOffline(final HRegionInfo hri) {
@@ -361,7 +470,11 @@ public class RegionStates {
     ServerName oldServerName = regionAssignments.remove(hri);
     if (oldServerName != null) {
       LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
-      serverHoldings.get(oldServerName).remove(hri);
+      Set<HRegionInfo> oldRegions = serverHoldings.get(oldServerName);
+      oldRegions.remove(hri);
+      if (oldRegions.isEmpty()) {
+        serverHoldings.remove(oldServerName);
+      }
     }
   }
 
@@ -370,19 +483,38 @@ public class RegionStates {
    */
   public synchronized List<HRegionInfo> serverOffline(
       final ZooKeeperWatcher watcher, final ServerName sn) {
-    // Clean up this server from map of servers to regions, and remove all regions
-    // of this server from online map of regions.
+    // Offline all regions on this server not already in transition.
     List<HRegionInfo> rits = new ArrayList<HRegionInfo>();
-    Set<HRegionInfo> assignedRegions = serverHoldings.remove(sn);
+    Set<HRegionInfo> assignedRegions = serverHoldings.get(sn);
     if (assignedRegions == null) {
       assignedRegions = new HashSet<HRegionInfo>();
     }
 
+    // Offline regions outside the loop to avoid ConcurrentModificationException
+    Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
     for (HRegionInfo region : assignedRegions) {
-      regionAssignments.remove(region);
+      // Offline open regions, no need to offline if SPLIT/MERGED/OFFLINE
+      if (isRegionOnline(region)) {
+        regionsToOffline.add(region);
+      } else {
+        RegionState state = getRegionState(region);
+        if (state.isSplitting() || state.isMerging()) {
+          LOG.debug("Offline splitting/merging region " + state);
+          try {
+            // Delete the ZNode if exists
+            ZKAssign.deleteNodeFailSilent(watcher, region);
+            regionsToOffline.add(region);
+          } catch (KeeperException ke) {
+            server.abort("Unexpected ZK exception deleting node " + region, ke);
+          }
+        }
+      }
+    }
+
+    for (HRegionInfo hri : regionsToOffline) {
+      regionOffline(hri);
     }
 
-    Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
     for (RegionState state : regionsInTransition.values()) {
       HRegionInfo hri = state.getRegion();
       if (assignedRegions.contains(hri)) {
@@ -390,36 +522,22 @@ public class RegionStates {
         // This region must be moving away from this server, or splitting/merging.
         // SSH will handle it, either skip assigning, or re-assign.
         LOG.info("Transitioning " + state + " will be handled by SSH for " + sn);
-        if (state.isSplitting() || state.isMerging()) {
-          LOG.info("Offline splitting/merging region " + state);
-          try {
-            // Delete the ZNode if exists
-            ZKAssign.deleteNodeFailSilent(watcher, hri);
-            // Offline regions outside the loop to avoid ConcurrentModificationException
-            regionsToOffline.add(hri);
-          } catch (KeeperException ke) {
-            server.abort("Unexpected ZK exception deleting node " + hri, ke);
-          }
-        }
       } else if (sn.equals(state.getServerName())) {
         // Region is in transition on this region server, and this
         // region is not open on this server. So the region must be
         // moving to this server from another one (i.e. opening or
-        // pending open on this server, was open on another one
-        if (state.isPendingOpen() || state.isOpening()) {
+        // pending open on this server, was open on another one.
+        // It could be in failed_close state too if tried several times
+        // to open it while the server is not reachable.
+        if (state.isPendingOpenOrOpening() || state.isFailedClose()) {
           LOG.info("Found opening region " + state + " to be reassigned by SSH for " + sn);
           rits.add(hri);
         } else {
-          LOG.warn("THIS SHOULD NOT HAPPEN: unexpected state "
-            + state + " of region in transition on server " + sn);
+          LOG.warn("THIS SHOULD NOT HAPPEN: unexpected " + state);
         }
       }
     }
-    for (HRegionInfo hri : regionsToOffline) {
-      regionOffline(hri);
-    }
 
-    assignedRegions.clear();
     this.notifyAll();
     return rits;
   }
@@ -470,24 +588,58 @@ public class RegionStates {
   }
 
   /**
-   * Waits until the specified region has completed assignment.
-   * <p>
-   * If the region is already assigned, returns immediately.  Otherwise, method
-   * blocks until the region is assigned.
-   */
-  public synchronized void waitForAssignment(
-      final HRegionInfo hri) throws InterruptedException {
-    if (!isRegionAssigned(hri)) return;
-
-    while(!server.isStopped() && !isRegionAssigned(hri)) {
-      RegionState rs = getRegionState(hri);
-      LOG.info("Waiting on " + rs + " to be assigned");
-      waitForUpdate(100);
-    }
-
-    if (server.isStopped()) {
-      LOG.info("Giving up wait on region " +
-        "assignment because stoppable.isStopped is set");
+   * Checking if a region was assigned to a server which is not online now.
+   * If so, we should hold re-assign this region till SSH has split its hlogs.
+   * Once logs are split, the last assignment of this region will be reset,
+   * which means a null last assignment server is ok for re-assigning.
+   *
+   * A region server could be dead but we don't know it yet. We may
+   * think it's online falsely. Therefore if a server is online, we still
+   * need to confirm it reachable and having the expected start code.
+   */
+  synchronized boolean wasRegionOnDeadServer(final String regionName) {
+    ServerName server = lastAssignments.get(regionName);
+    return isServerDeadAndNotProcessed(server);
+  }
+
+  synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
+    if (server == null) return false;
+    if (serverManager.isServerOnline(server)) {
+      String hostAndPort = server.getHostAndPort();
+      long startCode = server.getStartcode();
+      Long deadCode = deadServers.get(hostAndPort);
+      if (deadCode == null || startCode > deadCode.longValue()) {
+        if (serverManager.isServerReachable(server)) {
+          return false;
+        }
+        // The size of deadServers won't grow unbounded.
+        deadServers.put(hostAndPort, Long.valueOf(startCode));
+      }
+      // Watch out! If the server is not dead, the region could
+      // remain unassigned. That's why ServerManager#isServerReachable
+      // should use some retry.
+      //
+      // We cache this info since it is very unlikely for that
+      // instance to come back up later on. We don't want to expire
+      // the server since we prefer to let it die naturally.
+      LOG.warn("Couldn't reach online server " + server);
+    }
+    // Now, we know it's dead. Check if it's processed
+    return !processedServers.containsKey(server);
+  }
+
+ /**
+   * Get the last region server a region was on for purpose of re-assignment,
+   * i.e. should the re-assignment be held back till log split is done?
+   */
+  synchronized ServerName getLastRegionServerOfRegion(final String regionName) {
+    return lastAssignments.get(regionName);
+  }
+
+  synchronized void setLastRegionServerOfRegions(
+      final ServerName serverName, final List<HRegionInfo> regionInfos) {
+    for (HRegionInfo hri: regionInfos) {
+      lastAssignments.put(hri.getEncodedName(), serverName);
     }
   }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
Tue Oct  1 22:02:01 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Triple;
@@ -180,6 +181,7 @@ public class ServerManager {
     this(master, services, true);
   }
 
+  @SuppressWarnings("deprecation")
   ServerManager(final Server master, final MasterServices services,
       final boolean connect) throws IOException {
     this.master = master;
@@ -722,6 +724,29 @@ public class ServerManager {
     ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
   }
 
+  /**
+   * Check if a region server is reachable and has the expected start code
+   */
+  public boolean isServerReachable(ServerName server) {
+    if (server == null) throw new NullPointerException("Passed server is null");
+    int maximumAttempts = Math.max(1, master.getConfiguration().getInt(
+      "hbase.master.maximum.ping.server.attempts", 10));
+    for (int i = 0; i < maximumAttempts; i++) {
+      try {
+        AdminService.BlockingInterface admin = getRsAdmin(server);
+        if (admin != null) {
+          ServerInfo info = ProtobufUtil.getServerInfo(admin);
+          return info != null && info.hasServerName()
+            && server.getStartcode() == info.getServerName().getStartCode();
+        }
+      } catch (IOException ioe) {
+        LOG.debug("Couldn't reach " + server + ", try=" + i
+          + " of " + maximumAttempts, ioe);
+      }
+    }
+    return false;
+  }
+
     /**
     * @param sn
     * @return Admin interface for the remote regionserver named <code>sn</code>

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
Tue Oct  1 22:02:01 2013
@@ -98,7 +98,7 @@ public class ClosedRegionHandler extends
     }
     // ZK Node is in CLOSED state, assign it.
     assignmentManager.getRegionStates().updateRegionState(
-      regionInfo, RegionState.State.CLOSED, null);
+      regionInfo, RegionState.State.CLOSED);
     // This below has to do w/ online enable/disable of a table
     assignmentManager.removeClosedRegion(regionInfo);
     assignmentManager.assign(regionInfo, true);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
Tue Oct  1 22:02:01 2013
@@ -222,7 +222,7 @@ public class EnableTableHandler extends 
     for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
       HRegionInfo hri = regionLocation.getFirst();
       ServerName sn = regionLocation.getSecond();
-      if (!regionStates.isRegionInTransition(hri) && !regionStates.isRegionAssigned(hri))
{
+      if (regionStates.isRegionOffline(hri)) {
         if (this.retainAssignment && sn != null && serverManager.isServerOnline(sn))
{
           this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null,
sn));
         }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
Tue Oct  1 22:02:01 2013
@@ -19,9 +19,7 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
@@ -64,7 +62,8 @@ public class MetaServerShutdownHandler e
           } else {
             this.services.getMasterFileSystem().splitMetaLog(serverName);
           }
-        } 
+          am.getRegionStates().logSplit(HRegionInfo.FIRST_META_REGIONINFO);
+        }
       } catch (IOException ioe) {
         this.services.getExecutorService().submit(this);
         this.deadServers.add(serverName);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
Tue Oct  1 22:02:01 2013
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
+import java.util.concurrent.locks.Lock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -141,8 +142,9 @@ public class ServerShutdownHandler exten
       // If AssignmentManager hasn't finished rebuilding user regions,
       // we are not ready to assign dead regions either. So we re-queue up
       // the dead server for further processing too.
+      AssignmentManager am = services.getAssignmentManager();
       if (isCarryingMeta() // hbase:meta
-          || !services.getAssignmentManager().isFailoverCleanupDone()) {
+          || !am.isFailoverCleanupDone()) {
         this.services.getServerManager().processDeadServer(serverName, this.shouldSplitHlog);
         return;
       }
@@ -174,7 +176,7 @@ public class ServerShutdownHandler exten
           throw new IOException("Interrupted", e);
         } catch (IOException ioe) {
           LOG.info("Received exception accessing hbase:meta during server shutdown of " +
-              serverName + ", retrying hbase:meta read", ioe);
+            serverName + ", retrying hbase:meta read", ioe);
         }
       }
       if (this.server.isStopped()) {
@@ -192,6 +194,7 @@ public class ServerShutdownHandler exten
           } else {
             this.services.getMasterFileSystem().splitLog(serverName);
           }
+          am.getRegionStates().logSplit(serverName);
         } else {
           LOG.info("Skipping log splitting for " + serverName);
         }
@@ -203,7 +206,6 @@ public class ServerShutdownHandler exten
       // doing after log splitting.  Could do some states before -- OPENING?
       // OFFLINE? -- and then others after like CLOSING that depend on log
       // splitting.
-      AssignmentManager am = services.getAssignmentManager();
       List<HRegionInfo> regionsInTransition = am.processServerShutdown(serverName);
       LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) +
         " region(s) that " + (serverName == null? "null": serverName)  +
@@ -221,52 +223,56 @@ public class ServerShutdownHandler exten
           if (regionsInTransition.contains(hri)) {
             continue;
           }
-          RegionState rit = regionStates.getRegionTransitionState(hri);
-          if (processDeadRegion(hri, e.getValue(), am, server.getCatalogTracker())) {
-            ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri);
-            if (addressFromAM != null && !addressFromAM.equals(this.serverName))
{
-              // If this region is in transition on the dead server, it must be
-              // opening or pending_open, which should have been covered by AM#processServerShutdown
-              LOG.info("Skip assigning region " + hri.getRegionNameAsString()
-                + " because it has been opened in " + addressFromAM.getServerName());
-              continue;
-            }
-            if (rit != null) {
-              if (!rit.isOnServer(serverName)
-                  || rit.isClosed() || rit.isOpened()) {
-                // Skip regions that are in transition on other server,
-                // or in state closed/opened
-                LOG.info("Skip assigning region " + rit);
+          String encodedName = hri.getEncodedName();
+          Lock lock = am.acquireRegionLock(encodedName);
+          try {
+            RegionState rit = regionStates.getRegionTransitionState(hri);
+            if (processDeadRegion(hri, e.getValue(), am, server.getCatalogTracker())) {
+              ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri);
+              if (addressFromAM != null && !addressFromAM.equals(this.serverName))
{
+                // If this region is in transition on the dead server, it must be
+                // opening or pending_open, which should have been covered by AM#processServerShutdown
+                LOG.info("Skip assigning region " + hri.getRegionNameAsString()
+                  + " because it has been opened in " + addressFromAM.getServerName());
                 continue;
               }
-              try{
-                //clean zk node
-                LOG.info("Reassigning region with rs = " + rit + " and deleting zk node if
exists");
-                ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), hri);
-              } catch (KeeperException ke) {
-                this.server.abort("Unexpected ZK exception deleting unassigned node " + hri,
ke);
-                return;
+              if (rit != null) {
+                if (rit.getServerName() != null && !rit.isOnServer(serverName)) {
+                  // Skip regions that are in transition on other server
+                  LOG.info("Skip assigning region in transition on other server" + rit);
+                  continue;
+                }
+                try{
+                  //clean zk node
+                  LOG.info("Reassigning region with rs = " + rit + " and deleting zk node
if exists");
+                  ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), hri);
+                } catch (KeeperException ke) {
+                  this.server.abort("Unexpected ZK exception deleting unassigned node " +
hri, ke);
+                  return;
+                }
+              }
+              toAssignRegions.add(hri);
+            } else if (rit != null) {
+              if (rit.isPendingCloseOrClosing()
+                  && am.getZKTable().isDisablingOrDisabledTable(hri.getTable()))
{
+                // If the table was partially disabled and the RS went down, we should clear
the RIT
+                // and remove the node for the region.
+                // The rit that we use may be stale in case the table was in DISABLING state
+                // but though we did assign we will not be clearing the znode in CLOSING
state.
+                // Doing this will have no harm. See HBASE-5927
+                am.deleteClosingOrClosedNode(hri);
+                am.regionOffline(hri);
+              } else {
+                LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition "
+                  + rit + " not to be assigned by SSH of server " + serverName);
               }
             }
-            toAssignRegions.add(hri);
-          } else if (rit != null) {
-            if ((rit.isClosing() || rit.isPendingClose())
-                && am.getZKTable().isDisablingOrDisabledTable(hri.getTable())) {
-              // If the table was partially disabled and the RS went down, we should clear
the RIT
-              // and remove the node for the region.
-              // The rit that we use may be stale in case the table was in DISABLING state
-              // but though we did assign we will not be clearing the znode in CLOSING state.
-              // Doing this will have no harm. See HBASE-5927
-              am.deleteClosingOrClosedNode(hri);
-              am.regionOffline(hri);
-            } else {
-              LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition "
-                + rit + " not to be assigned by SSH of server " + serverName);
-            }
+          } finally {
+            lock.unlock();
           }
         }
       }
- 
+
       try {
         am.assign(toAssignRegions);
       } catch (InterruptedException ie) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Tue
Oct  1 22:02:01 2013
@@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.client.Ro
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -331,6 +332,18 @@ public class HBaseFsck extends Configure
       }
     }
 
+    errors.print("Average load: " + status.getAverageLoad());
+    errors.print("Number of requests: " + status.getRequestsCount());
+    errors.print("Number of regions: " + status.getRegionsCount());
+
+    Map<String, RegionState> rits = status.getRegionsInTransition();
+    errors.print("Number of regions in transition: " + rits.size());
+    if (details) {
+      for (RegionState state: rits.values()) {
+        errors.print("  " + state.toDescriptiveString());
+      }
+    }
+
     // Determine what's deployed
     processRegionServers(regionServers);
   }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java
Tue Oct  1 22:02:01 2013
@@ -150,7 +150,7 @@ public class TestDrainingServer {
 
     am.waitForAssignment(REGIONINFO);
 
-    assertTrue(am.getRegionStates().isRegionAssigned(REGIONINFO));
+    assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
     assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
   }
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
Tue Oct  1 22:02:01 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.HConstant
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
@@ -58,6 +59,7 @@ import org.apache.hadoop.hbase.util.Envi
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -75,15 +77,15 @@ public class TestAssignmentManagerOnClus
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    // Using the mock load balancer to control region plans
+    // Using the our load balancer to control region plans
     conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
-      MockLoadBalancer.class, LoadBalancer.class);
+      MyLoadBalancer.class, LoadBalancer.class);
     conf.setClass(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
-      MockRegionObserver.class, RegionObserver.class);
+      MyRegionObserver.class, RegionObserver.class);
     // Reduce the maximum attempts to speed up the test
     conf.setInt("hbase.assignment.maximum.attempts", 3);
 
-    TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.startMiniCluster(1, 4, null, MyMaster.class, null);
     admin = TEST_UTIL.getHBaseAdmin();
   }
 
@@ -306,12 +308,12 @@ public class TestAssignmentManagerOnClus
       AssignmentManager am = master.getAssignmentManager();
       assertTrue(am.waitForAssignment(hri));
 
-      MockRegionObserver.preCloseEnabled.set(true);
+      MyRegionObserver.preCloseEnabled.set(true);
       am.unassign(hri);
       RegionState state = am.getRegionStates().getRegionState(hri);
       assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
 
-      MockRegionObserver.preCloseEnabled.set(false);
+      MyRegionObserver.preCloseEnabled.set(false);
       am.unassign(hri, true);
 
       // region is closing now, will be re-assigned automatically.
@@ -327,7 +329,7 @@ public class TestAssignmentManagerOnClus
         getRegionStates().getRegionServerOfRegion(hri);
       TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
     } finally {
-      MockRegionObserver.preCloseEnabled.set(false);
+      MyRegionObserver.preCloseEnabled.set(false);
       TEST_UTIL.deleteTable(Bytes.toBytes(table));
     }
   }
@@ -353,12 +355,12 @@ public class TestAssignmentManagerOnClus
       AssignmentManager am = master.getAssignmentManager();
       assertTrue(am.waitForAssignment(hri));
 
-      MockRegionObserver.preCloseEnabled.set(true);
+      MyRegionObserver.preCloseEnabled.set(true);
       am.unassign(hri);
       RegionState state = am.getRegionStates().getRegionState(hri);
       assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
 
-      MockRegionObserver.preCloseEnabled.set(false);
+      MyRegionObserver.preCloseEnabled.set(false);
       am.unassign(hri, true);
 
       // region may still be assigned now since it's closing,
@@ -371,7 +373,7 @@ public class TestAssignmentManagerOnClus
         getRegionStates().getRegionServerOfRegion(hri);
       TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
     } finally {
-      MockRegionObserver.preCloseEnabled.set(false);
+      MyRegionObserver.preCloseEnabled.set(false);
       TEST_UTIL.deleteTable(Bytes.toBytes(table));
     }
   }
@@ -392,7 +394,7 @@ public class TestAssignmentManagerOnClus
         desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
       MetaEditor.addRegionToMeta(meta, hri);
 
-      MockLoadBalancer.controledRegion = hri.getEncodedName();
+      MyLoadBalancer.controledRegion = hri.getEncodedName();
 
       HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
       master.assignRegion(hri);
@@ -404,7 +406,7 @@ public class TestAssignmentManagerOnClus
       // Failed to open since no plan, so it's on no server
       assertNull(state.getServerName());
 
-      MockLoadBalancer.controledRegion = null;
+      MyLoadBalancer.controledRegion = null;
       master.assignRegion(hri);
       assertTrue(am.waitForAssignment(hri));
 
@@ -412,7 +414,7 @@ public class TestAssignmentManagerOnClus
         getRegionStates().getRegionServerOfRegion(hri);
       TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
     } finally {
-      MockLoadBalancer.controledRegion = null;
+      MyLoadBalancer.controledRegion = null;
       TEST_UTIL.deleteTable(Bytes.toBytes(table));
     }
   }
@@ -535,7 +537,7 @@ public class TestAssignmentManagerOnClus
       AssignmentManager am = master.getAssignmentManager();
       assertTrue(am.waitForAssignment(hri));
 
-      MockRegionObserver.postCloseEnabled.set(true);
+      MyRegionObserver.postCloseEnabled.set(true);
       am.unassign(hri);
       // Now region should pending_close or closing
       // Unassign it again forcefully so that we can trigger already
@@ -547,7 +549,7 @@ public class TestAssignmentManagerOnClus
 
       // Let region closing move ahead. The region should be closed
       // properly and re-assigned automatically
-      MockRegionObserver.postCloseEnabled.set(false);
+      MyRegionObserver.postCloseEnabled.set(false);
 
       // region may still be assigned now since it's closing,
       // let's check if it's assigned after it's out of transition
@@ -559,7 +561,7 @@ public class TestAssignmentManagerOnClus
         getRegionStates().getRegionServerOfRegion(hri);
       TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
     } finally {
-      MockRegionObserver.postCloseEnabled.set(false);
+      MyRegionObserver.postCloseEnabled.set(false);
       TEST_UTIL.deleteTable(Bytes.toBytes(table));
     }
   }
@@ -580,14 +582,14 @@ public class TestAssignmentManagerOnClus
         desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
       MetaEditor.addRegionToMeta(meta, hri);
 
-      MockRegionObserver.postOpenEnabled.set(true);
-      MockRegionObserver.postOpenCalled = false;
+      MyRegionObserver.postOpenEnabled.set(true);
+      MyRegionObserver.postOpenCalled = false;
       HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
       // Region will be opened, but it won't complete
       master.assignRegion(hri);
       long end = EnvironmentEdgeManager.currentTimeMillis() + 20000;
       // Wait till postOpen is called
-      while (!MockRegionObserver.postOpenCalled ) {
+      while (!MyRegionObserver.postOpenCalled ) {
         assertFalse("Timed out waiting for postOpen to be called",
           EnvironmentEdgeManager.currentTimeMillis() > end);
         Thread.sleep(300);
@@ -604,7 +606,7 @@ public class TestAssignmentManagerOnClus
       // Let's forcefully re-assign it to trigger closing/opening
       // racing. This test is to make sure this scenario
       // is handled properly.
-      MockRegionObserver.postOpenEnabled.set(false);
+      MyRegionObserver.postOpenEnabled.set(false);
       am.assign(hri, true, true);
 
       // let's check if it's assigned after it's out of transition
@@ -617,12 +619,124 @@ public class TestAssignmentManagerOnClus
       assertFalse("Region should assigned on a new region server",
         oldServerName.equals(serverName));
     } finally {
-      MockRegionObserver.postOpenEnabled.set(false);
+      MyRegionObserver.postOpenEnabled.set(false);
       TEST_UTIL.deleteTable(Bytes.toBytes(table));
     }
   }
 
-  static class MockLoadBalancer extends StochasticLoadBalancer {
+  /**
+   * Test force unassign/assign a region hosted on a dead server
+   */
+  @Test (timeout=60000)
+  public void testAssignRacingWithSSH() throws Exception {
+    String table = "testAssignRacingWithSSH";
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    MyMaster master = null;
+    try {
+      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
+      desc.addFamily(new HColumnDescriptor(FAMILY));
+      admin.createTable(desc);
+
+      HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
+      HRegionInfo hri = new HRegionInfo(
+        desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
+      MetaEditor.addRegionToMeta(meta, hri);
+
+      // Assign the region
+      master = (MyMaster)cluster.getMaster();
+      master.assignRegion(hri);
+
+      // Hold SSH before killing the hosting server
+      master.enableSSH(false);
+
+      // Kill the hosting server
+      AssignmentManager am = master.getAssignmentManager();
+      RegionStates regionStates = am.getRegionStates();
+      assertTrue(am.waitForAssignment(hri));
+      RegionState state = regionStates.getRegionState(hri);
+      ServerName oldServerName = state.getServerName();
+      cluster.killRegionServer(oldServerName);
+      cluster.waitForRegionServerToStop(oldServerName, -1);
+
+      // You can't assign a dead region before SSH
+      am.assign(hri, true, true);
+      state = regionStates.getRegionState(hri);
+      assertTrue(state.isFailedClose());
+
+      // You can't unassign a dead region before SSH either
+      am.unassign(hri, true);
+      state = regionStates.getRegionState(hri);
+      assertTrue(state.isFailedClose());
+
+      synchronized (regionStates) {
+        // Enable SSH so that log can be split
+        master.enableSSH(true);
+
+        // We hold regionStates now, so logSplit
+        // won't be known to AM yet.
+        am.unassign(hri, true);
+        state = regionStates.getRegionState(hri);
+        assertTrue(state.isOffline());
+      }
+
+      // let's check if it's assigned after it's out of transition.
+      // no need to assign it manually, SSH should do it
+      am.waitOnRegionToClearRegionsInTransition(hri);
+      assertTrue(am.waitForAssignment(hri));
+
+      ServerName serverName = master.getAssignmentManager().
+        getRegionStates().getRegionServerOfRegion(hri);
+      TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
+    } finally {
+      if (master != null) {
+        master.enableSSH(true);
+      }
+      TEST_UTIL.deleteTable(Bytes.toBytes(table));
+    }
+  }
+
+  /**
+   * Test force unassign/assign a region of a disabled table
+   */
+  @Test (timeout=60000)
+  public void testAssignDisabledRegion() throws Exception {
+    String table = "testAssignDisabledRegion";
+    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+    MyMaster master = null;
+    try {
+      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
+      desc.addFamily(new HColumnDescriptor(FAMILY));
+      admin.createTable(desc);
+
+      HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
+      HRegionInfo hri = new HRegionInfo(
+        desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
+      MetaEditor.addRegionToMeta(meta, hri);
+
+      // Assign the region
+      master = (MyMaster)cluster.getMaster();
+      master.assignRegion(hri);
+      AssignmentManager am = master.getAssignmentManager();
+      RegionStates regionStates = am.getRegionStates();
+      assertTrue(am.waitForAssignment(hri));
+
+      // Disable the table
+      admin.disableTable(table);
+      assertTrue(regionStates.isRegionOffline(hri));
+
+      // You can't assign a disabled region
+      am.assign(hri, true, true);
+      assertTrue(regionStates.isRegionOffline(hri));
+
+      // You can't unassign a disabled region either
+      am.unassign(hri, true);
+      assertTrue(regionStates.isRegionOffline(hri));
+    } finally {
+      TEST_UTIL.deleteTable(Bytes.toBytes(table));
+    }
+  }
+
+  static class MyLoadBalancer extends StochasticLoadBalancer {
     // For this region, if specified, always assign to nowhere
     static volatile String controledRegion = null;
 
@@ -636,7 +750,28 @@ public class TestAssignmentManagerOnClus
     }
   }
 
-  public static class MockRegionObserver extends BaseRegionObserver {
+  public static class MyMaster extends HMaster {
+    AtomicBoolean enabled = new AtomicBoolean(true);
+
+    public MyMaster(Configuration conf) throws IOException, KeeperException,
+        InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    public boolean isServerShutdownHandlerEnabled() {
+      return enabled.get() && super.isServerShutdownHandlerEnabled();
+    }
+
+    public void enableSSH(boolean enabled) {
+      this.enabled.set(enabled);
+      if (enabled) {
+        serverManager.processQueuedDeadServers();
+      }
+    }
+  }
+
+  public static class MyRegionObserver extends BaseRegionObserver {
     // If enabled, fail all preClose calls
     static AtomicBoolean preCloseEnabled = new AtomicBoolean(false);
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java?rev=1528227&r1=1528226&r2=1528227&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
Tue Oct  1 22:02:01 2013
@@ -167,6 +167,7 @@ public class TestSplitTransactionOnClust
     return hri;
   }
 
+  @SuppressWarnings("deprecation")
   @Test(timeout = 60000)
   public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception
{
     final TableName tableName =
@@ -209,6 +210,8 @@ public class TestSplitTransactionOnClust
       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
       try {
         secondSplit = true;
+        // make region splittable
+        region.initialize();
         st.prepare();
         st.execute(regionServer, regionServer);
       } catch (IOException e) {
@@ -933,8 +936,8 @@ public class TestSplitTransactionOnClust
       assertTrue("not able to find a splittable region", region != null);
       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
         @Override
-        int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName)
-            throws KeeperException, IOException {
+        int createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region,
+            ServerName serverName) throws KeeperException, IOException {
           throw new SplittingNodeCreationFailedException ();
         }
       };



Mime
View raw message