hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject [5/5] git commit: HBASE-11059 ZK-less region assignment
Date Sat, 14 Jun 2014 15:40:40 GMT
HBASE-11059 ZK-less region assignment


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58549428
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58549428
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58549428

Branch: refs/heads/master
Commit: 58549428a66550aed83bfb6f1da405decc1b0f61
Parents: 047463c
Author: Jimmy Xiang <jxiang@cloudera.com>
Authored: Wed Apr 23 16:56:36 2014 -0700
Committer: Jimmy Xiang <jxiang@cloudera.com>
Committed: Sat Jun 14 08:39:29 2014 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HRegionInfo.java    |    1 +
 .../org/apache/hadoop/hbase/RegionLoad.java     |    4 +-
 .../apache/hadoop/hbase/catalog/MetaReader.java |   10 +-
 .../apache/hadoop/hbase/master/RegionState.java |   37 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java       |    4 +-
 .../org/apache/hadoop/hbase/HConstants.java     |   10 +
 .../generated/RegionServerStatusProtos.java     | 2959 +++++++++++++++++-
 .../src/main/protobuf/RegionServerStatus.proto  |   49 +
 .../hadoop/hbase/master/AssignCallable.java     |    7 +-
 .../hadoop/hbase/master/AssignmentManager.java  |  782 ++++-
 .../hadoop/hbase/master/MasterRpcServices.java  |   35 +-
 .../hadoop/hbase/master/RegionStateStore.java   |  226 ++
 .../hadoop/hbase/master/RegionStates.java       |  295 +-
 .../hadoop/hbase/master/ServerManager.java      |   12 +-
 .../hadoop/hbase/master/UnAssignCallable.java   |   47 +
 .../master/handler/ServerShutdownHandler.java   |   27 +-
 .../hbase/regionserver/HRegionServer.java       |   94 +-
 .../regionserver/RegionMergeTransaction.java    |  111 +-
 .../regionserver/RegionServerServices.java      |   11 +
 .../hbase/regionserver/SplitTransaction.java    |   53 +-
 .../handler/CloseRegionHandler.java             |   15 +-
 .../regionserver/handler/OpenRegionHandler.java |   46 +-
 .../apache/hadoop/hbase/util/ConfigUtil.java    |   33 +
 .../hadoop/hbase/MockRegionServerServices.java  |   15 +-
 .../apache/hadoop/hbase/TestDrainingServer.java |    1 +
 .../client/TestScannersFromClientSide.java      |    5 +-
 .../hadoop/hbase/master/MockRegionServer.java   |   14 +-
 .../hbase/master/TestAssignmentManager.java     |    4 +-
 .../master/TestAssignmentManagerOnCluster.java  |   42 +-
 .../hadoop/hbase/master/TestMasterFailover.java |  104 +
 .../hbase/master/TestOpenedRegionHandler.java   |    2 +
 .../hadoop/hbase/master/TestRestartCluster.java |    1 +
 .../master/TestZKBasedOpenCloseRegion.java      |    1 +
 .../hbase/master/TestZKLessAMOnCluster.java     |   42 +
 .../TestEndToEndSplitTransaction.java           |    8 +-
 .../TestRegionMergeTransactionOnCluster.java    |   20 +-
 .../regionserver/TestRegionServerNoMaster.java  |    1 +
 .../TestSplitTransactionOnCluster.java          |  134 +-
 .../regionserver/TestZKLessMergeOnCluster.java  |   45 +
 .../regionserver/TestZKLessSplitOnCluster.java  |   45 +
 .../handler/TestCloseRegionHandler.java         |    1 +
 .../handler/TestOpenRegionHandler.java          |    1 +
 .../apache/hadoop/hbase/util/TestHBaseFsck.java |    6 +
 .../zookeeper/TestRecoverableZooKeeper.java     |    2 +-
 44 files changed, 4907 insertions(+), 455 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 78e175d..cc87c49 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -1048,6 +1048,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
    * @param r Result to pull from
    * @return A ServerName instance or null if necessary fields not found or empty.
    */
+  // TODO: ServerName should be a better place for this method?
   public static ServerName getServerName(final Result r) {
     Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
     if (cell == null || cell.getValueLength() == 0) return null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index 2fa45e4..28e6d6b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -203,6 +203,8 @@ public class RegionLoad {
     }
     sb = Strings.appendKeyValue(sb, "compactionProgressPct",
         compactionProgressPct);
+    sb = Strings.appendKeyValue(sb, "completeSequenceId",
+        this.getCompleteSequenceId());
     return sb.toString();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 4f15390..ec0f4a0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -160,6 +160,7 @@ public class MetaReader {
    * @return An {@link HTable} for <code>tableName</code>
    * @throws IOException
    */
+  @SuppressWarnings("deprecation")
   private static HTable getHTable(final CatalogTracker catalogTracker,
       final TableName tableName)
   throws IOException {
@@ -482,8 +483,11 @@ public class MetaReader {
       @Override
       void add(Result r) {
         if (r == null || r.isEmpty()) return;
+        if (HRegionInfo.getHRegionInfo(r) == null) return;
         ServerName sn = HRegionInfo.getServerName(r);
-        if (sn != null && sn.equals(serverName)) this.results.add(r);
+        if (sn != null && sn.equals(serverName)) {
+          this.results.add(r);
+        }
       }
     };
     fullScan(catalogTracker, v);
@@ -491,8 +495,8 @@ public class MetaReader {
     if (results != null && !results.isEmpty()) {
       // Convert results to Map keyed by HRI
       for (Result r: results) {
-        Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
-        if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
+        HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
+        if (hri != null) hris.put(hri, r);
       }
     }
     return hris;

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
index 1170387..3289ac1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.Date;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -35,7 +32,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
  * So it is almost immutable.
  */
 @InterfaceAudience.Private
-public class RegionState implements org.apache.hadoop.io.Writable {
+public class RegionState {
 
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
@@ -77,6 +74,11 @@ public class RegionState implements org.apache.hadoop.io.Writable {
   }
 
   public RegionState(HRegionInfo region,
+      State state, ServerName serverName) {
+    this(region, state, System.currentTimeMillis(), serverName);
+  }
+
+  public RegionState(HRegionInfo region,
       State state, long stamp, ServerName serverName) {
     this.hri = region;
     this.state = state;
@@ -383,25 +385,26 @@ public class RegionState implements org.apache.hadoop.io.Writable {
   }
 
   /**
-   * @deprecated Writables are going away
+   * Check if two states are the same, except timestamp
    */
-  @Deprecated
   @Override
-  public void readFields(DataInput in) throws IOException {
-    hri = new HRegionInfo();
-    hri.readFields(in);
-    state = State.valueOf(in.readUTF());
-    stamp.set(in.readLong());
+  public boolean equals(Object obj) {
+    if (this == obj) return true;
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    RegionState tmp = (RegionState)obj;
+    return tmp.hri.equals(hri) && tmp.state == state
+      && ((serverName != null && serverName.equals(tmp.serverName))
+        || (tmp.serverName == null && serverName == null));
   }
 
   /**
-   * @deprecated Writables are going away
+   * Don't count timestamp in hash code calculation
    */
-  @Deprecated
   @Override
-  public void write(DataOutput out) throws IOException {
-    hri.write(out);
-    out.writeUTF(state.name());
-    out.writeLong(stamp.get());
+  public int hashCode() {
+    return (serverName != null ? serverName.hashCode() * 11 : 0)
+      + hri.hashCode() + 5 * state.ordinal();
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 52981e4..6b9c8ba 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -171,7 +171,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable
{
     try {
       // Create all the necessary "directories" of znodes
       ZKUtil.createWithParents(this, baseZNode);
-      ZKUtil.createAndFailSilent(this, assignmentZNode);
+      if (conf.getBoolean("hbase.assignment.usezk", true)) {
+        ZKUtil.createAndFailSilent(this, assignmentZNode);
+      }
       ZKUtil.createAndFailSilent(this, rsZNode);
       ZKUtil.createAndFailSilent(this, drainingZNode);
       ZKUtil.createAndFailSilent(this, tableZNode);

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3bf1249..b27679c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -405,6 +405,16 @@ public final class HConstants {
   /** The open seqnum column qualifier */
   public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes("seqnumDuringOpen");
 
+  /** The state column qualifier */
+  public static final byte [] STATE_QUALIFIER = Bytes.toBytes("state");
+
+  /**
+   * The serverName column qualifier. Its the server where the region is
+   * transitioning on, while column server is the server where the region is
+   * opened on. They are the same when the region is in state OPEN.
+   */
+  public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes("sn");
+
   /** The lower-half split region column qualifier */
   public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
 


Mime
View raw message