hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [05/12] git commit: HBASE-11546 Backport ZK-less region assignment to 0.98 (Virag Kothari) [1/8]
Date Tue, 26 Aug 2014 15:15:23 GMT
HBASE-11546 Backport ZK-less region assignment to 0.98 (Virag Kothari) [1/8]

HBASE-11059 ZK-less region assignment (Jimmy Xiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e6ffa86e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e6ffa86e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e6ffa86e

Branch: refs/heads/0.98
Commit: e6ffa86e33ee173afcff15ca4b614e6ec56357ed
Parents: 5aa39bb
Author: Andrew Purtell <apurtell@apache.org>
Authored: Tue Aug 26 08:01:09 2014 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Tue Aug 26 08:03:10 2014 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HRegionInfo.java    |    1 +
 .../org/apache/hadoop/hbase/RegionLoad.java     |    4 +-
 .../apache/hadoop/hbase/catalog/MetaReader.java |   10 +-
 .../apache/hadoop/hbase/master/RegionState.java |   32 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java       |    4 +-
 .../org/apache/hadoop/hbase/HConstants.java     |   10 +
 .../generated/RegionServerStatusProtos.java     | 2959 +++++++++++++++++-
 .../src/main/protobuf/RegionServerStatus.proto  |   49 +
 .../hadoop/hbase/master/AssignCallable.java     |    7 +-
 .../hadoop/hbase/master/AssignmentManager.java  |  778 ++++-
 .../org/apache/hadoop/hbase/master/HMaster.java |   29 +
 .../hadoop/hbase/master/RegionStateStore.java   |  225 ++
 .../hadoop/hbase/master/RegionStates.java       |  296 +-
 .../hadoop/hbase/master/ServerManager.java      |   12 +-
 .../master/handler/ServerShutdownHandler.java   |   27 +-
 .../hbase/regionserver/HRegionServer.java       |   97 +-
 .../regionserver/RegionMergeTransaction.java    |  111 +-
 .../regionserver/RegionServerServices.java      |   11 +
 .../hbase/regionserver/SplitTransaction.java    |   70 +-
 .../handler/CloseRegionHandler.java             |   30 +-
 .../regionserver/handler/OpenRegionHandler.java |   42 +-
 .../apache/hadoop/hbase/util/ConfigUtil.java    |   33 +
 .../hadoop/hbase/MockRegionServerServices.java  |   15 +-
 .../apache/hadoop/hbase/TestDrainingServer.java |    1 +
 .../client/TestScannersFromClientSide.java      |    5 +-
 .../hadoop/hbase/master/MockRegionServer.java   |   12 +
 .../hbase/master/TestAssignmentManager.java     |    2 +
 .../master/TestAssignmentManagerOnCluster.java  |   37 +-
 .../hadoop/hbase/master/TestMasterFailover.java |  103 +
 .../hbase/master/TestOpenedRegionHandler.java   |    2 +
 .../hadoop/hbase/master/TestRestartCluster.java |    1 +
 .../master/TestZKBasedOpenCloseRegion.java      |    1 +
 .../hbase/master/TestZKLessAMOnCluster.java     |   42 +
 .../TestEndToEndSplitTransaction.java           |    7 +-
 .../TestRegionMergeTransactionOnCluster.java    |   20 +-
 .../regionserver/TestRegionServerNoMaster.java  |    1 +
 .../TestSplitTransactionOnCluster.java          |  130 +-
 .../regionserver/TestZKLessMergeOnCluster.java  |   45 +
 .../regionserver/TestZKLessSplitOnCluster.java  |   45 +
 .../handler/TestCloseRegionHandler.java         |    1 +
 .../handler/TestOpenRegionHandler.java          |    1 +
 .../apache/hadoop/hbase/util/TestHBaseFsck.java |    6 +
 .../zookeeper/TestRecoverableZooKeeper.java     |    2 +-
 43 files changed, 4854 insertions(+), 462 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 502688c..da2ac0d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -1048,6 +1048,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
    * @param r Result to pull from
    * @return A ServerName instance or null if necessary fields not found or empty.
    */
+  // TODO: ServerName should be a better place for this method?
   public static ServerName getServerName(final Result r) {
     Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
     if (cell == null || cell.getValueLength() == 0) return null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index 2fa45e4..28e6d6b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -203,6 +203,8 @@ public class RegionLoad {
     }
     sb = Strings.appendKeyValue(sb, "compactionProgressPct",
         compactionProgressPct);
+    sb = Strings.appendKeyValue(sb, "completeSequenceId",
+        this.getCompleteSequenceId());
     return sb.toString();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index 4f15390..ec0f4a0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -160,6 +160,7 @@ public class MetaReader {
    * @return An {@link HTable} for <code>tableName</code>
    * @throws IOException
    */
+  @SuppressWarnings("deprecation")
   private static HTable getHTable(final CatalogTracker catalogTracker,
       final TableName tableName)
   throws IOException {
@@ -482,8 +483,11 @@ public class MetaReader {
       @Override
       void add(Result r) {
         if (r == null || r.isEmpty()) return;
+        if (HRegionInfo.getHRegionInfo(r) == null) return;
         ServerName sn = HRegionInfo.getServerName(r);
-        if (sn != null && sn.equals(serverName)) this.results.add(r);
+        if (sn != null && sn.equals(serverName)) {
+          this.results.add(r);
+        }
       }
     };
     fullScan(catalogTracker, v);
@@ -491,8 +495,8 @@ public class MetaReader {
     if (results != null && !results.isEmpty()) {
       // Convert results to Map keyed by HRI
       for (Result r: results) {
-        Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
-        if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
+        HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
+        if (hri != null) hris.put(hri, r);
       }
     }
     return hris;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
index 5b07e88..e6aa279 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java
@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.hbase.master;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.Date;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -35,7 +32,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
  * So it is almost immutable.
  */
 @InterfaceAudience.Private
-public class RegionState implements org.apache.hadoop.io.Writable {
+public class RegionState {
 
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
@@ -77,6 +74,11 @@ public class RegionState implements org.apache.hadoop.io.Writable {
   }
 
   public RegionState(HRegionInfo region,
+      State state, ServerName serverName) {
+    this(region, state, System.currentTimeMillis(), serverName);
+  }
+
+  public RegionState(HRegionInfo region,
       State state, long stamp, ServerName serverName) {
     this.hri = region;
     this.state = state;
@@ -398,25 +400,11 @@ public class RegionState implements org.apache.hadoop.io.Writable {
   }
 
   /**
-   * @deprecated Writables are going away
-   */
-  @Deprecated
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    hri = new HRegionInfo();
-    hri.readFields(in);
-    state = State.valueOf(in.readUTF());
-    stamp.set(in.readLong());
-  }
-
-  /**
-   * @deprecated Writables are going away
+   * Don't count timestamp in hash code calculation
    */
-  @Deprecated
   @Override
-  public void write(DataOutput out) throws IOException {
-    hri.write(out);
-    out.writeUTF(state.name());
-    out.writeLong(stamp.get());
+  public int hashCode() {
+    return (serverName != null ? serverName.hashCode() * 11 : 0)
+      + hri.hashCode() + 5 * state.ordinal();
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 2c88a3c..d32f6fd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -172,7 +172,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable
{
     try {
       // Create all the necessary "directories" of znodes
       ZKUtil.createWithParents(this, baseZNode);
-      ZKUtil.createAndFailSilent(this, assignmentZNode);
+      if (conf.getBoolean("hbase.assignment.usezk", true)) {
+        ZKUtil.createAndFailSilent(this, assignmentZNode);
+      }
       ZKUtil.createAndFailSilent(this, rsZNode);
       ZKUtil.createAndFailSilent(this, drainingZNode);
       ZKUtil.createAndFailSilent(this, tableZNode);

http://git-wip-us.apache.org/repos/asf/hbase/blob/e6ffa86e/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 379ae7b..cba4471 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -395,6 +395,16 @@ public final class HConstants {
   /** The open seqnum column qualifier */
   public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes("seqnumDuringOpen");
 
+  /** The state column qualifier */
+  public static final byte [] STATE_QUALIFIER = Bytes.toBytes("state");
+
+  /**
+   * The serverName column qualifier. Its the server where the region is
+   * transitioning on, while column server is the server where the region is
+   * opened on. They are the same when the region is in state OPEN.
+   */
+  public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes("sn");
+
   /** The lower-half split region column qualifier */
   public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
 


Mime
View raw message