hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject svn commit: r1565041 [1/3] - in /hbase/branches/hbase-10070: hbase-client/src/main/java/org/apache/hadoop/hbase/ hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ hbase-client/src/tes...
Date Thu, 06 Feb 2014 02:04:54 GMT
Author: enis
Date: Thu Feb  6 02:04:53 2014
New Revision: 1565041

URL: http://svn.apache.org/r1565041
Log:
HBASE-10347 HRegionInfo changes for adding replicaId and MetaEditor/MetaReader changes for region replicas

Added:
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java
    hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java
Modified:
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java
    hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
    hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
    hbase/branches/hbase-10070/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/hbase-10070/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
    hbase/branches/hbase-10070/hbase-protocol/src/main/protobuf/HBase.proto
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java

Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Thu Feb  6 02:04:53 2014
@@ -29,7 +29,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.google.protobuf.HBaseZeroCopyByteString;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -47,15 +46,34 @@ import org.apache.hadoop.hbase.util.Pair
 import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.io.DataInputBuffer;
 
+import com.google.protobuf.HBaseZeroCopyByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 /**
- * HRegion information.
- * Contains HRegion id, start and end keys, a reference to this HRegions' table descriptor, etc.
+ * Information about a region. A region is a range of keys in the whole keyspace of a table, an
+ * identifier (a timestamp) for differentiating between subset ranges (after region split)
+ * and a replicaId for differentiating the instance for the same range and some status information
+ * about the region.
+ *
+ * The region has a unique name which consists of the following fields:
+ * <li> tableName   : The name of the table </li>
+ * <li> startKey    : The startKey for the region. </li>
+ * <li> regionId    : A timestamp when the region is created. </li>
+ * <li> replicaId   : An id starting from 0 to differentiate replicas of the same region range
+ * but hosted in separated servers. The same region range can be hosted in multiple locations.</li>
+ * <li> encodedName : An MD5 encoded string for the region name.</li>
  *
- * On a big cluster, each client will have thousands of instances of this object, often
- *  100 000 of them if not million. It's important to keep the object size as small
- *  as possible.
+ * <br> Other than the fields in the region name, region info contains:
+ * <li> endKey      : the endKey for the region (exclusive) </li>
+ * <li> split       : Whether the region is split </li>
+ * <li> offline     : Whether the region is offline </li>
+ *
+ * In 0.98 or before, a list of table's regions would fully cover the total keyspace, and at any
+ * point in time, a row key always belongs to a single region, which is hosted in a single server.
+ * In 0.99+, a region can have multiple instances (called replicas), and thus a range (or row) can
+ * correspond to multiple HRegionInfo's. These HRI's share the same fields however except the
+ * replicaId field. If the replicaId is not set, it defaults to 0, which is compatible with the
+ * previous behavior of a range corresponding to 1 region.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
@@ -118,6 +136,14 @@ public class HRegionInfo implements Comp
   /** A non-capture group so that this can be embedded. */
   public static final String ENCODED_REGION_NAME_REGEX = "(?:[a-f0-9]+)";
 
+  // to keep appended int's sorted in string format. Only allows 2 bytes to be
+  // sorted for replicaId
+  public static final String REPLICA_ID_FORMAT = "%04X";
+
+  public static final byte REPLICA_ID_DELIMITER = (byte)'_';
+
+  private static final int MAX_REPLICA_ID = 0xFFFF;
+  private static final int DEFAULT_REPLICA_ID = 0;
   /**
    * Does region name contain its encoded name?
    * @param regionName region name
@@ -190,6 +216,7 @@ public class HRegionInfo implements Comp
   public static final String NO_HASH = null;
   private String encodedName = null;
   private byte [] encodedNameAsBytes = null;
+  private int replicaId = DEFAULT_REPLICA_ID;
 
   // Current TableName
   private TableName tableName = null;
@@ -205,6 +232,7 @@ public class HRegionInfo implements Comp
     result ^= Arrays.hashCode(this.endKey);
     result ^= Boolean.valueOf(this.offLine).hashCode();
     result ^= Arrays.hashCode(this.tableName.getName());
+    result ^= this.replicaId;
     this.hashCode = result;
   }
 
@@ -248,7 +276,6 @@ public class HRegionInfo implements Comp
     this(tableName, startKey, endKey, false);
   }
 
-
   /**
    * Construct HRegionInfo with explicit parameters
    *
@@ -265,7 +292,6 @@ public class HRegionInfo implements Comp
     this(tableName, startKey, endKey, split, System.currentTimeMillis());
   }
 
-
   /**
    * Construct HRegionInfo with explicit parameters
    *
@@ -280,7 +306,25 @@ public class HRegionInfo implements Comp
   public HRegionInfo(final TableName tableName, final byte[] startKey,
                      final byte[] endKey, final boolean split, final long regionid)
   throws IllegalArgumentException {
+    this(tableName, startKey, endKey, split, regionid, DEFAULT_REPLICA_ID);
+  }
 
+  /**
+   * Construct HRegionInfo with explicit parameters
+   *
+   * @param tableName the table descriptor
+   * @param startKey first key in region
+   * @param endKey end of key range
+   * @param split true if this region has split and we have daughter regions
+   * regions that may or may not hold references to this region.
+   * @param regionid Region id to use.
+   * @param replicaId the replicaId to use
+   * @throws IllegalArgumentException
+   */
+  public HRegionInfo(final TableName tableName, final byte[] startKey,
+                     final byte[] endKey, final boolean split, final long regionid,
+                     final int replicaId)
+    throws IllegalArgumentException {
     super();
     if (tableName == null) {
       throw new IllegalArgumentException("TableName cannot be null");
@@ -288,8 +332,12 @@ public class HRegionInfo implements Comp
     this.tableName = tableName;
     this.offLine = false;
     this.regionId = regionid;
+    this.replicaId = replicaId;
+    if (this.replicaId > MAX_REPLICA_ID) {
+      throw new IllegalArgumentException("ReplicaId cannot be greater than" + MAX_REPLICA_ID);
+    }
 
-    this.regionName = createRegionName(this.tableName, startKey, regionId, true);
+    this.regionName = createRegionName(this.tableName, startKey, regionId, replicaId, true);
 
     this.split = split;
     this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
@@ -315,8 +363,14 @@ public class HRegionInfo implements Comp
     this.hashCode = other.hashCode();
     this.encodedName = other.getEncodedName();
     this.tableName = other.tableName;
+    this.replicaId = other.replicaId;
   }
 
+  public HRegionInfo(HRegionInfo other, int replicaId) {
+    this(other);
+    this.replicaId = replicaId;
+    this.setHashCode();
+  }
 
   /**
    * Make a region name of passed parameters.
@@ -350,6 +404,22 @@ public class HRegionInfo implements Comp
    * Make a region name of passed parameters.
    * @param tableName
    * @param startKey Can be null
+   * @param regionid Region id (Usually timestamp from when region was created).
+   * @param replicaId
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
+   * @return Region name made of passed tableName, startKey, id and replicaId
+   */
+  public static byte [] createRegionName(final TableName tableName,
+      final byte [] startKey, final long regionid, int replicaId, boolean newFormat) {
+    return createRegionName(tableName, startKey, Bytes.toBytes(Long.toString(regionid)),
+        replicaId, newFormat);
+  }
+
+  /**
+   * Make a region name of passed parameters.
+   * @param tableName
+   * @param startKey Can be null
    * @param id Region id (Usually timestamp from when region was created).
    * @param newFormat should we create the region name in the new format
    *                  (such that it contains its encoded name?).
@@ -357,9 +427,35 @@ public class HRegionInfo implements Comp
    */
   public static byte [] createRegionName(final TableName tableName,
       final byte [] startKey, final byte [] id, boolean newFormat) {
-    byte [] b = new byte [tableName.getName().length + 2 + id.length +
-       (startKey == null? 0: startKey.length) +
-       (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+    return createRegionName(tableName, startKey, id, DEFAULT_REPLICA_ID, newFormat);
+  }
+  /**
+   * Make a region name of passed parameters.
+   * @param tableName
+   * @param startKey Can be null
+   * @param id Region id (Usually timestamp from when region was created).
+   * @param replicaId
+   * @param newFormat should we create the region name in the new format
+   * @return Region name made of passed tableName, startKey, id and replicaId
+   */
+  public static byte [] createRegionName(final TableName tableName,
+      final byte [] startKey, final byte [] id, final int replicaId, boolean newFormat) {
+    int len = tableName.getName().length + 2 + id.length +
+        (startKey == null? 0: startKey.length);
+    if (newFormat) {
+      len += MD5_HEX_LENGTH + 2;
+    }
+    byte[] replicaIdBytes = null;
+    // Special casing: replicaId is only appended if replicaId is greater than
+    // 0. This is because all regions in meta would have to be migrated to the new
+    // name otherwise
+    if (replicaId > 0) {
+      // use string representation for replica id
+      replicaIdBytes = Bytes.toBytes(String.format(REPLICA_ID_FORMAT, replicaId));
+      len += 1 + replicaIdBytes.length;
+    }
+
+    byte [] b = new byte [len];
 
     int offset = tableName.getName().length;
     System.arraycopy(tableName.getName(), 0, b, 0, offset);
@@ -372,11 +468,17 @@ public class HRegionInfo implements Comp
     System.arraycopy(id, 0, b, offset, id.length);
     offset += id.length;
 
+    if (replicaIdBytes != null) {
+      b[offset++] = REPLICA_ID_DELIMITER;
+      System.arraycopy(replicaIdBytes, 0, b, offset, replicaIdBytes.length);
+      offset += replicaIdBytes.length;
+    }
+
     if (newFormat) {
       //
       // Encoded name should be built into the region name.
       //
-      // Use the region name thus far (namely, <tablename>,<startKey>,<id>)
+      // Use the region name thus far (namely, <tablename>,<startKey>,<id>_<replicaId>)
       // to compute a MD5 hash to be used as the encoded name, and append
       // it to the byte buffer.
       //
@@ -447,6 +549,11 @@ public class HRegionInfo implements Comp
    */
   public static byte [][] parseRegionName(final byte [] regionName)
   throws IOException {
+    // Region name is of the format:
+    // tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]
+    // startkey can contain the delimiter (',') so we parse from the start and end
+
+    // parse from start
     int offset = -1;
     for (int i = 0; i < regionName.length; i++) {
       if (regionName[i] == HConstants.DELIMITER) {
@@ -458,8 +565,27 @@ public class HRegionInfo implements Comp
     byte[] tableName = new byte[offset];
     System.arraycopy(regionName, 0, tableName, 0, offset);
     offset = -1;
-    for (int i = regionName.length - 1; i > 0; i--) {
-      if(regionName[i] == HConstants.DELIMITER) {
+
+    int endOffset = regionName.length;
+    // check whether regionName contains encodedName
+    if (regionName.length > MD5_HEX_LENGTH + 2
+        && regionName[regionName.length-1] == ENC_SEPARATOR
+        && regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) {
+      endOffset = endOffset - MD5_HEX_LENGTH - 2;
+    }
+
+    // parse from end
+    byte[] replicaId = null;
+    int idEndOffset = endOffset;
+    for (int i = endOffset - 1; i > 0; i--) {
+      if (regionName[i] == REPLICA_ID_DELIMITER) { //replicaId may or may not be present
+        replicaId = new byte[endOffset - i - 1];
+        System.arraycopy(regionName, i + 1, replicaId, 0,
+          endOffset - i - 1);
+        idEndOffset = i;
+        // do not break, continue to search for id
+      }
+      if (regionName[i] == HConstants.DELIMITER) {
         offset = i;
         break;
       }
@@ -471,13 +597,17 @@ public class HRegionInfo implements Comp
       System.arraycopy(regionName, tableName.length + 1, startKey, 0,
           offset - tableName.length - 1);
     }
-    byte [] id = new byte[regionName.length - offset - 1];
+    byte [] id = new byte[idEndOffset - offset - 1];
     System.arraycopy(regionName, offset + 1, id, 0,
-        regionName.length - offset - 1);
-    byte [][] elements = new byte[3][];
+      idEndOffset - offset - 1);
+    byte [][] elements = new byte[replicaId == null ? 3 : 4][];
     elements[0] = tableName;
     elements[1] = startKey;
     elements[2] = id;
+    if (replicaId != null) {
+      elements[3] = replicaId;
+    }
+
     return elements;
   }
 
@@ -631,7 +761,6 @@ public class HRegionInfo implements Comp
     this.offLine = offLine;
   }
 
-
   /**
    * @return True if this is a split parent region.
    */
@@ -644,6 +773,14 @@ public class HRegionInfo implements Comp
   }
 
   /**
+   * Returns the region replica id
+   * @return returns region replica id
+   */
+  public int getReplicaId() {
+    return replicaId;
+  }
+
+  /**
    * @see java.lang.Object#toString()
    */
   @Override
@@ -654,7 +791,8 @@ public class HRegionInfo implements Comp
       Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" +
       Bytes.toStringBinary(this.endKey) + "'" +
       (isOffline()? ", OFFLINE => true": "") +
-      (isSplit()? ", SPLIT => true": "") + "}";
+      (isSplit()? ", SPLIT => true": "") +
+      ((replicaId > 0)? ", REPLICA_ID => " + replicaId : "") + "}";
   }
 
   /**
@@ -766,6 +904,7 @@ public class HRegionInfo implements Comp
   // Comparable
   //
 
+  @Override
   public int compareTo(HRegionInfo o) {
     if (o == null) {
       return 1;
@@ -806,6 +945,9 @@ public class HRegionInfo implements Comp
       return -1;
     }
 
+    int replicaDiff = this.getReplicaId() - o.getReplicaId();
+    if (replicaDiff != 0) return replicaDiff;
+
     if (this.offLine == o.offLine)
       return 0;
     if (this.offLine == true) return -1;
@@ -849,6 +991,7 @@ public class HRegionInfo implements Comp
     }
     builder.setOffline(info.isOffline());
     builder.setSplit(info.isSplit());
+    builder.setReplicaId(info.getReplicaId());
     return builder.build();
   }
 
@@ -866,6 +1009,7 @@ public class HRegionInfo implements Comp
       return FIRST_META_REGIONINFO;
     }
     long regionId = proto.getRegionId();
+    int replicaId = proto.hasReplicaId() ? proto.getReplicaId() : DEFAULT_REPLICA_ID;
     byte[] startKey = null;
     byte[] endKey = null;
     if (proto.hasStartKey()) {
@@ -881,7 +1025,7 @@ public class HRegionInfo implements Comp
     HRegionInfo hri = new HRegionInfo(
         tableName,
         startKey,
-        endKey, split, regionId);
+        endKey, split, regionId, replicaId);
     if (proto.hasOffline()) {
       hri.setOffline(proto.getOffline());
     }
@@ -980,7 +1124,9 @@ public class HRegionInfo implements Comp
    * @return A pair of the {@link HRegionInfo} and the {@link ServerName}
    * (or null for server address if no address set in hbase:meta).
    * @throws IOException
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static Pair<HRegionInfo, ServerName> getHRegionInfoAndServerName(final Result r) {
     HRegionInfo info =
       getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
@@ -994,7 +1140,9 @@ public class HRegionInfo implements Comp
    * table Result.
    * @param data a Result object from the catalog table scan
    * @return HRegionInfo or null
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static HRegionInfo getHRegionInfo(Result data) {
     return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
   }
@@ -1005,21 +1153,25 @@ public class HRegionInfo implements Comp
    * @param data a Result object from the catalog table scan
    * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
    * parent
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
     HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
     HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
 
     return new PairOfSameType<HRegionInfo>(splitA, splitB);
   }
-  
+
   /**
    * Returns the merge regions by reading the corresponding columns of the catalog table
    * Result.
    * @param data a Result object from the catalog table scan
    * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
    * parent
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
     HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
     HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
@@ -1035,7 +1187,9 @@ public class HRegionInfo implements Comp
    * {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or
    * {@link HConstants#REGIONINFO_QUALIFIER}.
    * @return An HRegionInfo instance or null.
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
     Cell cell = r.getColumnLatestCell(
         HConstants.CATALOG_FAMILY, qualifier);
@@ -1044,10 +1198,9 @@ public class HRegionInfo implements Comp
   }
 
   /**
-   * Returns a {@link ServerName} from catalog table {@link Result}.
-   * @param r Result to pull from
-   * @return A ServerName instance or null if necessary fields not found or empty.
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static ServerName getServerName(final Result r) {
     Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
     if (cell == null || cell.getValueLength() == 0) return null;
@@ -1065,7 +1218,9 @@ public class HRegionInfo implements Comp
    * E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
    * @param r Result to pull the seqNum from
    * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
+   * @deprecated use MetaReader methods for interacting with meta layouts
    */
+  @Deprecated
   public static long getSeqNumDuringOpen(final Result r) {
     Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER);
     if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
@@ -1185,5 +1340,4 @@ public class HRegionInfo implements Comp
     }
     return false;
   }
-
 }

Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java Thu Feb  6 02:04:53 2014
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hbase;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hbase.util.Addressing;
 
 /**
@@ -32,9 +31,10 @@ import org.apache.hadoop.hbase.util.Addr
  * On a big cluster, each client will have thousands of instances of this object, often
  *  100 000 of them if not million. It's important to keep the object size as small
  *  as possible.
+ * <br>This interface has been marked InterfaceAudience.Public in 0.96 and 0.98, it is
+ * no longer the case.
  */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
 public class HRegionLocation implements Comparable<HRegionLocation> {
   private final HRegionInfo regionInfo;
   private final ServerName serverName;
@@ -112,6 +112,7 @@ public class HRegionLocation implements 
     return serverName;
   }
 
+  @Override
   public int compareTo(HRegionLocation o) {
     return serverName.compareTo(o.getServerName());
   }

Added: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java?rev=1565041&view=auto
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java (added)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java Thu Feb  6 02:04:53 2014
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Container for holding a list of {@link HRegionLocation}'s that correspond to the
+ * same range. The list is indexed by the replicaId. This is an immutable list,
+ * however mutation operations are provided which returns a new List via copy-on-write
+ * (assuming small number of locations)
+ */
+@InterfaceAudience.Private
+public class RegionLocations {
+
+  private final int numNonNullElements;
+  private final HRegionLocation[] locations; // replicaId -> HRegionLocation.
+
+  /**
+   * Constructs the region location list. The locations array should
+   * contain all the locations for known replicas for the region, and should be
+   * sorted in replicaId ascending order.
+   * @param locations an array of HRegionLocations for the same region range
+   */
+  public RegionLocations(HRegionLocation... locations) {
+    int numNonNullElements = 0;
+    int maxReplicaId = -1;
+    for (HRegionLocation loc : locations) {
+      if (loc != null) {
+        numNonNullElements++;
+        if (loc.getRegionInfo().getReplicaId() > maxReplicaId) {
+          maxReplicaId = loc.getRegionInfo().getReplicaId();
+        }
+      }
+    }
+    this.numNonNullElements = numNonNullElements;
+
+    if (maxReplicaId + 1 == locations.length) {
+      this.locations = locations;
+    } else {
+      this.locations = new HRegionLocation[maxReplicaId + 1];
+      for (HRegionLocation loc : locations) {
+        if (loc != null) {
+          this.locations[loc.getRegionInfo().getReplicaId()] = loc;
+        }
+      }
+    }
+  }
+
+  public RegionLocations(Collection<HRegionLocation> locations) {
+    this(locations.toArray(new HRegionLocation[locations.size()]));
+  }
+
+  /**
+   * Returns the size of the list even if some of the elements
+   * might be null.
+   * @return the size of the list (corresponding to the max replicaId)
+   */
+  public int size() {
+    return locations.length;
+  }
+
+  /**
+   * Returns the size of not-null locations
+   * @return the size of not-null locations
+   */
+  public int numNonNullElements() {
+    return numNonNullElements;
+  }
+
+  /**
+   * Returns whether there are non-null elements in the list
+   * @return whether there are non-null elements in the list
+   */
+  public boolean isEmpty() {
+    return numNonNullElements == 0;
+  }
+
+  /**
+   * Returns a new HRegionLocationList with the locations removed (set to null)
+   * which have the destination server as given.
+   * @param serverName the serverName to remove locations of
+   * @return an HRegionLocationList object with removed locations or the same object
+   * if nothing is removed
+   */
+  public RegionLocations removeByServer(ServerName serverName) {
+    HRegionLocation[] newLocations = null;
+    for (int i = 0; i < locations.length; i++) {
+      // check whether something to remove
+      if (locations[i] != null && serverName.equals(locations[i].getServerName())) {
+        if (newLocations == null) { //first time
+          newLocations = new HRegionLocation[locations.length];
+          System.arraycopy(locations, 0, newLocations, 0, i);
+        }
+        newLocations[i] = null;
+      } else if (newLocations != null) {
+        newLocations[i] = locations[i];
+      }
+    }
+    return newLocations == null ? this : new RegionLocations(newLocations);
+  }
+
+  /**
+   * Removes the given location from the list
+   * @param location the location to remove
+   * @return an HRegionLocationList object with removed locations or the same object
+   * if nothing is removed
+   */
+  public RegionLocations remove(HRegionLocation location) {
+    HRegionLocation[] newLocations = null;
+    for (int i = 0; i < locations.length; i++) {
+      // check whether something to remove. HRL.compareTo() compares ONLY the
+      // serverName. We want to compare the HRI's as well.
+      if (locations[i] != null
+          && location.getRegionInfo().equals(locations[i].getRegionInfo())
+          && location.equals(locations[i])) {
+        if (newLocations == null) { //first time
+          newLocations = new HRegionLocation[locations.length];
+          System.arraycopy(locations, 0, newLocations, 0, i);
+        }
+        newLocations[i] = null;
+      } else if (newLocations != null) {
+        newLocations[i] = locations[i];
+      }
+    }
+    return newLocations == null ? this : new RegionLocations(newLocations);
+  }
+
+  /**
+   * Merges this HRegionLocation list with the given list assuming
+   * same range, and keeping the most up to date version of the
+   * HRegionLocation entries from either list according to seqNum. If seqNums
+   * are equal, the location from the argument (other) is taken.
+   * @param other the locations to merge with
+   * @return an HRegionLocationList object with merged locations or the same object
+   * if nothing is merged
+   */
+  public RegionLocations mergeLocations(RegionLocations other) {
+    assert other != null;
+
+    HRegionLocation[] newLocations = null;
+
+    int max = Math.max(this.locations.length, other.locations.length);
+
+    for (int i = 0; i < max; i++) {
+      HRegionLocation thisLoc = this.getRegionLocation(i);
+      HRegionLocation otherLoc = other.getRegionLocation(i);
+
+      HRegionLocation selectedLoc = selectRegionLocation(thisLoc,
+        otherLoc, true, false);
+
+      if (selectedLoc != thisLoc) {
+        if (newLocations == null) {
+          newLocations = new HRegionLocation[max];
+          System.arraycopy(locations, 0, newLocations, 0, i);
+        }
+      }
+      if (newLocations != null) {
+        newLocations[i] = selectedLoc;
+      }
+    }
+
+    return newLocations == null ? this : new RegionLocations(newLocations);
+  }
+
+  private HRegionLocation selectRegionLocation(HRegionLocation oldLocation,
+      HRegionLocation location, boolean checkForEquals, boolean force) {
+    if (location == null) {
+      return oldLocation == null ? null : oldLocation;
+    }
+
+    if (oldLocation == null) {
+      return location;
+    }
+
+    if (force
+        || isGreaterThan(location.getSeqNum(), oldLocation.getSeqNum(), checkForEquals)) {
+      return location;
+    }
+    return oldLocation;
+  }
+
+  /**
+   * Updates the location with new only if the new location has a higher
+   * seqNum than the old one or force is true.
+   * @param location the location to add or update
+   * @param checkForEquals whether to update the location if seqNums for the
+   * HRegionLocations for the old and new location are the same
+   * @param force whether to force update
+   * @return an HRegionLocationList object with updated locations or the same object
+   * if nothing is updated
+   */
+  public RegionLocations updateLocation(HRegionLocation location,
+      boolean checkForEquals, boolean force) {
+    assert location != null;
+
+    int replicaId = location.getRegionInfo().getReplicaId();
+
+    HRegionLocation oldLoc = getRegionLocation(location.getRegionInfo().getReplicaId());
+    HRegionLocation selectedLoc = selectRegionLocation(oldLoc, location,
+      checkForEquals, force);
+
+    if (selectedLoc == oldLoc) {
+      return this;
+    }
+    HRegionLocation[] newLocations = new HRegionLocation[Math.max(locations.length, replicaId +1)];
+    System.arraycopy(locations, 0, newLocations, 0, locations.length);
+    newLocations[replicaId] = location;
+    return new RegionLocations(newLocations);
+  }
+
+  private boolean isGreaterThan(long a, long b, boolean checkForEquals) {
+    return a > b || (checkForEquals && (a == b));
+  }
+
+  public HRegionLocation getRegionLocation(int replicaId) {
+    if (replicaId >= locations.length) {
+      return null;
+    }
+    return locations[replicaId];
+  }
+
+  /**
+   * Returns the region location from the list for matching regionName, which can
+   * be regionName or encodedRegionName
+   * @param regionName regionName or encodedRegionName
+   * @return HRegionLocation found or null
+   */
+  public HRegionLocation getRegionLocationByRegionName(byte[] regionName) {
+    for (HRegionLocation loc : locations) {
+      if (loc != null) {
+        if (Bytes.equals(loc.getRegionInfo().getRegionName(), regionName)
+            || Bytes.equals(loc.getRegionInfo().getEncodedNameAsBytes(), regionName)) {
+          return loc;
+        }
+      }
+    }
+    return null;
+  }
+
+  public HRegionLocation[] getRegionLocations() {
+    return locations;
+  }
+
+  /**
+   * Returns the first not-null region location in the list
+   */
+  public HRegionLocation getRegionLocation() {
+    for (HRegionLocation loc : locations) {
+      if (loc != null) {
+        return loc;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder builder = new StringBuilder("[");
+    for (HRegionLocation loc : locations) {
+      if (loc != null) {
+        if (builder.length() > 1) {
+          builder.append(", ");
+        }
+        builder.append(loc);
+      }
+    }
+    builder.append("]");
+    return builder.toString();
+  }
+
+}

Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Feb  6 02:04:53 2014
@@ -17,37 +17,83 @@
  */
 package org.apache.hadoop.hbase.catalog;
 
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PairOfSameType;
 
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
+import com.google.common.annotations.VisibleForTesting;
 
 /**
  * Reads region and assignment information from <code>hbase:meta</code>.
  */
 @InterfaceAudience.Private
 public class MetaReader {
+
+  /*
+   * HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the
+   * same table range (table, startKey, endKey). For every range, there will be at least one
+   * HRI defined which is called default replica.
+   *
+   * Meta layout (as of 0.98 + HBASE-10070) is like:
+   * For each table range, there is a single row, formatted like:
+   * <tableName>,<startKey>,<regionId>,<encodedRegionName>. This row corresponds to the regionName
+   * of the default region replica.
+   * Columns are:
+   * info:regioninfo         => contains serialized HRI for the default region replica
+   * info:server             => contains hostname:port (in string form) for the server hosting
+   *                            the default regionInfo replica
+   * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting the
+   *                            regionInfo replica with replicaId
+   * info:serverstartcode    => contains server start code (in binary long form) for the server
+   *                            hosting the default regionInfo replica
+   * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for the
+   *                                     server hosting the regionInfo replica with replicaId
+   * info:seqnumDuringOpen    => contains seqNum (in binary long form) for the region at the time
+   *                             the server opened the region with default replicaId
+   * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region at
+   *                             the time the server opened the region with replicaId
+   * info:splitA              => contains a serialized HRI for the first daughter region if the
+   *                             region is split
+   * info:splitB              => contains a serialized HRI for the second daughter region if the
+   *                             region is split
+   * info:mergeA              => contains a serialized HRI for the first parent region if the
+   *                             region is the result of a merge
+   * info:mergeB              => contains a serialized HRI for the second parent region if the
+   *                             region is the result of a merge
+   *
+   * The actual layout of meta should be encapsulated inside MetaReader and MetaEditor methods,
+   * and should not leak out of those (through Result objects, etc)
+   */
+
   // TODO: Strip CatalogTracker from this class.  Its all over and in the end
   // its only used to get its Configuration so we can get associated
   // Connection.
@@ -63,59 +109,12 @@ public class MetaReader {
       META_REGION_PREFIX, 0, len);
   }
 
-  /**
-   * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
-   * tables in the specified set of disabled tables.
-   * @param catalogTracker
-   * @param disabledTables set of disabled tables that will not be returned
-   * @return Returns a map of every region to it's currently assigned server,
-   * according to META.  If the region does not have an assignment it will have
-   * a null value in the map.
-   * @throws IOException
-   */
-  public static Map<HRegionInfo, ServerName> fullScan(
-      CatalogTracker catalogTracker, final Set<TableName> disabledTables)
-  throws IOException {
-    return fullScan(catalogTracker, disabledTables, false);
-  }
+  /** The delimiter for meta columns for replicaIds > 0 */
+  protected static final char META_REPLICA_ID_DELIMITER = '_';
 
-  /**
-   * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
-   * tables in the specified set of disabled tables.
-   * @param catalogTracker
-   * @param disabledTables set of disabled tables that will not be returned
-   * @param excludeOfflinedSplitParents If true, do not include offlined split
-   * parents in the return.
-   * @return Returns a map of every region to it's currently assigned server,
-   * according to META.  If the region does not have an assignment it will have
-   * a null value in the map.
-   * @throws IOException
-   */
-  public static Map<HRegionInfo, ServerName> fullScan(
-      CatalogTracker catalogTracker, final Set<TableName> disabledTables,
-      final boolean excludeOfflinedSplitParents)
-  throws IOException {
-    final Map<HRegionInfo, ServerName> regions =
-      new TreeMap<HRegionInfo, ServerName>();
-    Visitor v = new Visitor() {
-      @Override
-      public boolean visit(Result r) throws IOException {
-        if (r ==  null || r.isEmpty()) return true;
-        Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
-        HRegionInfo hri = region.getFirst();
-        if (hri  == null) return true;
-        if (hri.getTable() == null) return true;
-        if (disabledTables.contains(
-            hri.getTable())) return true;
-        // Are we to include split parents in the list?
-        if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
-        regions.put(hri, region.getSecond());
-        return true;
-      }
-    };
-    fullScan(catalogTracker, v);
-    return regions;
-  }
+  /** A regex for parsing server columns from meta. See above javadoc for meta layout */
+  private static final Pattern SERVER_COLUMN_PATTERN
+    = Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
 
   /**
    * Performs a full scan of <code>hbase:meta</code>.
@@ -206,33 +205,81 @@ public class MetaReader {
   }
 
   /**
-   * Reads the location of the specified region
+   * Gets the region info and assignment for the specified region.
    * @param catalogTracker
-   * @param regionName region whose location we are after
-   * @return location of region as a {@link ServerName} or null if not found
+   * @param regionName Region to lookup.
+   * @return Location and HRegionInfo for <code>regionName</code>
    * @throws IOException
+   * @deprecated use {@link #getRegionLocation(CatalogTracker, byte[])} instead
    */
-  static ServerName readRegionLocation(CatalogTracker catalogTracker,
-      byte [] regionName)
+  @Deprecated
+  public static Pair<HRegionInfo, ServerName> getRegion(
+      CatalogTracker catalogTracker, byte [] regionName)
   throws IOException {
-    Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
-    return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
+    HRegionLocation location = getRegionLocation(catalogTracker, regionName);
+    return location == null
+        ? null
+        : new Pair<HRegionInfo, ServerName>(location.getRegionInfo(), location.getServerName());
   }
 
   /**
-   * Gets the region info and assignment for the specified region.
+   * Returns the HRegionLocation from meta for the given region
    * @param catalogTracker
-   * @param regionName Region to lookup.
-   * @return Location and HRegionInfo for <code>regionName</code>
+   * @param regionName
+   * @return HRegionLocation for the given region
    * @throws IOException
    */
-  public static Pair<HRegionInfo, ServerName> getRegion(
-      CatalogTracker catalogTracker, byte [] regionName)
-  throws IOException {
-    Get get = new Get(regionName);
+  public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
+      byte[] regionName) throws IOException {
+    byte[] row = regionName;
+    HRegionInfo parsedInfo = null;
+    try {
+      parsedInfo = parseRegionInfoFromRegionName(regionName);
+      row = getMetaKeyForRegion(parsedInfo);
+    } catch (Exception parseEx) {
+      LOG.warn("Received parse exception:" + parseEx);
+    }
+    Get get = new Get(row);
     get.addFamily(HConstants.CATALOG_FAMILY);
     Result r = get(getCatalogHTable(catalogTracker), get);
-    return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
+    RegionLocations locations = getRegionLocations(r);
+    return locations == null
+        ? null
+        : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+  }
+
+  /**
+   * Returns the HRegionLocation from meta for the given region
+   * @param catalogTracker
+   * @param regionInfo
+   * @return HRegionLocation for the given region
+   * @throws IOException
+   */
+  public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
+      HRegionInfo regionInfo) throws IOException {
+    byte[] row = getMetaKeyForRegion(regionInfo);
+    Get get = new Get(row);
+    get.addFamily(HConstants.CATALOG_FAMILY);
+    Result r = get(getCatalogHTable(catalogTracker), get);
+    return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+  }
+
+  /** Returns the row key to use for this regionInfo */
+  protected static byte[] getMetaKeyForRegion(HRegionInfo regionInfo) {
+    return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
+  }
+
+  /** Returns an HRI parsed from this regionName. Not all the fields of the HRI
+   * is stored in the name, so the returned object should only be used for the fields
+   * in the regionName.
+   */
+  protected static HRegionInfo parseRegionInfoFromRegionName(byte[] regionName)
+      throws IOException {
+    byte[][] fields = HRegionInfo.parseRegionName(regionName);
+    long regionId =  Long.parseLong(Bytes.toString(fields[2]));
+    int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
+    return new HRegionInfo(
+      TableName.valueOf(fields[0]), fields[1], fields[1], false, regionId, replicaId);
   }
 
   /**
@@ -257,10 +304,8 @@ public class MetaReader {
   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
     Result result = getRegionResult(catalogTracker, regionName);
-    HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
-        HConstants.MERGEA_QUALIFIER);
-    HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
-        HConstants.MERGEB_QUALIFIER);
+    HRegionInfo mergeA = getHRegionInfo(result, HConstants.MERGEA_QUALIFIER);
+    HRegionInfo mergeB = getHRegionInfo(result, HConstants.MERGEB_QUALIFIER);
     if (mergeA == null && mergeB == null) {
       return null;
     }
@@ -288,8 +333,12 @@ public class MetaReader {
 
       @Override
       public boolean visit(Result r) throws IOException {
-        this.current =
-          HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
+        RegionLocations locations = getRegionLocations(r);
+        if (locations == null || locations.getRegionLocation().getRegionInfo() == null) {
+          LOG.warn("No serialized HRegionInfo in " + r);
+          return true;
+        }
+        this.current = locations.getRegionLocation().getRegionInfo();
         if (this.current == null) {
           LOG.warn("No serialized HRegionInfo in " + r);
           return true;
@@ -437,28 +486,33 @@ public class MetaReader {
     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
-      private Pair<HRegionInfo, ServerName> current = null;
+      private RegionLocations current = null;
 
       @Override
       public boolean visit(Result r) throws IOException {
-        HRegionInfo hri =
-          HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
-        if (hri == null) {
+        current = getRegionLocations(r);
+        if (current == null || current.getRegionLocation().getRegionInfo() == null) {
           LOG.warn("No serialized HRegionInfo in " + r);
           return true;
         }
+        HRegionInfo hri = current.getRegionLocation().getRegionInfo();
         if (!isInsideTable(hri, tableName)) return false;
         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
-        ServerName sn = HRegionInfo.getServerName(r);
-        // Populate this.current so available when we call #add
-        this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
         // Else call super and add this Result to the collection.
         return super.visit(r);
       }
 
       @Override
       void add(Result r) {
-        this.results.add(this.current);
+        if (current == null) {
+          return;
+        }
+        for (HRegionLocation loc : current.getRegionLocations()) {
+          if (loc != null) {
+            this.results.add(new Pair<HRegionInfo, ServerName>(
+                loc.getRegionInfo(), loc.getServerName()));
+          }
+        }
       }
     };
     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
@@ -482,19 +536,18 @@ public class MetaReader {
       @Override
       void add(Result r) {
         if (r == null || r.isEmpty()) return;
-        ServerName sn = HRegionInfo.getServerName(r);
-        if (sn != null && sn.equals(serverName)) this.results.add(r);
+        RegionLocations locations = getRegionLocations(r);
+        if (locations == null) return;
+        for (HRegionLocation loc : locations.getRegionLocations()) {
+          if (loc != null) {
+            if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
+              hris.put(loc.getRegionInfo(), r);
+            }
+          }
+        }
       }
     };
     fullScan(catalogTracker, v);
-    List<Result> results = v.getResults();
-    if (results != null && !results.isEmpty()) {
-      // Convert results to Map keyed by HRI
-      for (Result r: results) {
-        Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
-        if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
-      }
-    }
     return hris;
   }
 
@@ -505,8 +558,13 @@ public class MetaReader {
       public boolean visit(Result r) throws IOException {
         if (r ==  null || r.isEmpty()) return true;
         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
-        HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
-        LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
+        RegionLocations locations = getRegionLocations(r);
+        if (locations == null) return true;
+        for (HRegionLocation loc : locations.getRegionLocations()) {
+          if (loc != null) {
+            LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
+          }
+        }
         return true;
       }
     };
@@ -551,6 +609,215 @@ public class MetaReader {
   }
 
   /**
+   * Returns the column family used for meta columns.
+   * @return HConstants.CATALOG_FAMILY.
+   */
+  protected static byte[] getFamily() {
+    return HConstants.CATALOG_FAMILY;
+  }
+
+  /**
+   * Returns the column qualifier for serialized region info
+   * @return HConstants.REGIONINFO_QUALIFIER
+   */
+  protected static byte[] getRegionInfoColumn() {
+    return HConstants.REGIONINFO_QUALIFIER;
+  }
+
+  /**
+   * Returns the column qualifier for server column for replicaId
+   * @param replicaId the replicaId of the region
+   * @return a byte[] for server column qualifier
+   */
+  protected static byte[] getServerColumn(int replicaId) {
+    return replicaId == 0
+        ? HConstants.SERVER_QUALIFIER
+        : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+          + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+  }
+
+  /**
+   * Returns the column qualifier for server start code column for replicaId
+   * @param replicaId the replicaId of the region
+   * @return a byte[] for server start code column qualifier
+   */
+  protected static byte[] getStartCodeColumn(int replicaId) {
+    return replicaId == 0
+        ? HConstants.STARTCODE_QUALIFIER
+        : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+          + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+  }
+
+  /**
+   * Returns the column qualifier for seqNum column for replicaId
+   * @param replicaId the replicaId of the region
+   * @return a byte[] for seqNum column qualifier
+   */
+  protected static byte[] getSeqNumColumn(int replicaId) {
+    return replicaId == 0
+        ? HConstants.SEQNUM_QUALIFIER
+        : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+          + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+  }
+
+  /**
+   * Parses the replicaId from the server column qualifier. See top of the class javadoc
+   * for the actual meta layout
+   * @param serverColumn the column qualifier
+   * @return an int for the replicaId
+   */
+  @VisibleForTesting
+  static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
+    String serverStr = Bytes.toString(serverColumn);
+
+    Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
+    if (matcher.matches() && matcher.groupCount() > 0) {
+        String group = matcher.group(1);
+        if (group != null && group.length() > 0) {
+          return Integer.parseInt(group.substring(1), 16);
+        } else {
+          return 0;
+      }
+    }
+    return -1;
+  }
+
+  /**
+   * Returns a {@link ServerName} from catalog table {@link Result}.
+   * @param r Result to pull from
+   * @return A ServerName instance or null if necessary fields not found or empty.
+   */
+  private static ServerName getServerName(final Result r, final int replicaId) {
+    byte[] serverColumn = getServerColumn(replicaId);
+    Cell cell = r.getColumnLatestCell(getFamily(), serverColumn);
+    if (cell == null || cell.getValueLength() == 0) return null;
+    String hostAndPort = Bytes.toString(
+        cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+    byte[] startcodeColumn = getStartCodeColumn(replicaId);
+    cell = r.getColumnLatestCell(getFamily(), startcodeColumn);
+    if (cell == null || cell.getValueLength() == 0) return null;
+    return ServerName.valueOf(hostAndPort,
+      Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
+  }
+
+  /**
+   * The latest seqnum that the server writing to meta observed when opening the region.
+   * E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
+   * @param r Result to pull the seqNum from
+   * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
+   */
+  private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
+    Cell cell = r.getColumnLatestCell(getFamily(), getSeqNumColumn(replicaId));
+    if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
+    return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+  }
+
+  /**
+   * Returns an HRegionLocationList extracted from the result.
+   * @return an HRegionLocationList containing all locations for the region range
+   */
+  public static RegionLocations getRegionLocations(final Result r) {
+    if (r == null) return null;
+    HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
+    if (regionInfo == null) return null;
+
+    List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
+    NavigableMap<byte[],NavigableMap<byte[],byte[]>> familyMap = r.getNoVersionMap();
+
+    locations.add(getRegionLocation(r, regionInfo, 0));
+
+    NavigableMap<byte[], byte[]> infoMap = familyMap.get(getFamily());
+    if (infoMap == null) return new RegionLocations(locations);
+
+    // iterate until all serverName columns are seen
+    int replicaId = 0;
+    byte[] serverColumn = getServerColumn(replicaId);
+    SortedMap<byte[], byte[]> serverMap = infoMap.tailMap(serverColumn, false);
+    if (serverMap.isEmpty()) return new RegionLocations(locations);
+
+    for (Entry<byte[], byte[]> entry : serverMap.entrySet()) {
+      replicaId = parseReplicaIdFromServerColumn(entry.getKey());
+      if (replicaId < 0) {
+        break;
+      }
+
+      locations.add(getRegionLocation(r, regionInfo, replicaId));
+    }
+
+    return new RegionLocations(locations);
+  }
+
+  /**
+   * Returns the HRegionLocation parsed from the given meta row Result
+   * for the given regionInfo and replicaId. The regionInfo can be the default region info
+   * for the replica.
+   * @param r the meta row result
+   * @param regionInfo RegionInfo for default replica
+   * @param replicaId the replicaId for the HRegionLocation
+   * @return HRegionLocation parsed from the given meta row Result for the given replicaId
+   */
+  private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo,
+      final int replicaId) {
+    ServerName serverName = getServerName(r, replicaId);
+    long seqNum = getSeqNumDuringOpen(r, replicaId);
+    HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
+    return new HRegionLocation(replicaInfo, serverName, seqNum);
+  }
+
+  /**
+   * Returns HRegionInfo object from the column
+   * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
+   * table Result.
+   * @param data a Result object from the catalog table scan
+   * @return HRegionInfo or null
+   */
+  public static HRegionInfo getHRegionInfo(Result data) {
+    return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
+  }
+
+  /**
+   * Returns the HRegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and
+   * <code>qualifier</code> of the catalog table result.
+   * @param r a Result object from the catalog table scan
+   * @param qualifier Column family qualifier
+   * @return An HRegionInfo instance or null.
+   */
+  private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
+    Cell cell = r.getColumnLatestCell(getFamily(), qualifier);
+    if (cell == null) return null;
+    return HRegionInfo.parseFromOrNull(cell.getValueArray(),
+      cell.getValueOffset(), cell.getValueLength());
+  }
+
+  /**
+   * Returns the daughter regions by reading the corresponding columns of the catalog table
+   * Result.
+   * @param data a Result object from the catalog table scan
+   * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
+   * parent
+   */
+  public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
+    HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
+    HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
+
+    return new PairOfSameType<HRegionInfo>(splitA, splitB);
+  }
+
+  /**
+   * Returns the merge regions by reading the corresponding columns of the catalog table
+   * Result.
+   * @param data a Result object from the catalog table scan
+   * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
+   * parent
+   */
+  public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
+    HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
+    HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
+
+    return new PairOfSameType<HRegionInfo>(mergeA, mergeB);
+  }
+
+  /**
    * Implementations 'visit' a catalog table row.
    */
   public interface Visitor {

Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java Thu Feb  6 02:04:53 2014
@@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentMa
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -49,7 +48,6 @@ import org.apache.hadoop.hbase.TableName
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
 import org.cloudera.htrace.Trace;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -93,7 +91,7 @@ class AsyncProcess {
   private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
   protected static final AtomicLong COUNTER = new AtomicLong();
 
-  /** 
+  /**
    * The context used to wait for results from one submit call.
    * 1) If AsyncProcess is set to track errors globally, and not per call (for HTable puts),
    *    then errors and failed operations in this object will reflect global errors.
@@ -111,10 +109,15 @@ class AsyncProcess {
   /** Return value from a submit that didn't contain any requests. */
   private static final AsyncRequestFuture NO_REQS_RESULT = new AsyncRequestFuture() {
     public final Object[] result = new Object[0];
+    @Override
     public boolean hasError() { return false; }
+    @Override
     public RetriesExhaustedWithDetailsException getErrors() { return null; }
+    @Override
     public List<? extends Row> getFailedOperations() { return null; }
+    @Override
     public Object[] getResults() { return result; }
+    @Override
     public void waitUntilDone() throws InterruptedIOException {}
   };
 
@@ -685,7 +688,7 @@ class AsyncProcess {
       // Do not use the exception for updating cache because it might be coming from
       // any of the regions in the MultiAction.
       byte[] row = rsActions.actions.values().iterator().next().get(0).getAction().getRow();
-      hConnection.updateCachedLocations(tableName, row, null, server);
+      hConnection.updateCachedLocations(tableName, null, row, null, server);
       errorsByServer.reportServerError(server);
 
       List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
@@ -790,7 +793,7 @@ class AsyncProcess {
             if (!regionFailureRegistered) { // We're doing this once per location.
               regionFailureRegistered = true;
               // The location here is used as a server name.
-              hConnection.updateCachedLocations(tableName, row.getRow(), result, server);
+              hConnection.updateCachedLocations(tableName, regionName, row.getRow(), result, server);
               if (failureCount == 0) {
                 errorsByServer.reportServerError(server);
                 canRetry = errorsByServer.canRetryMore(numAttempt);
@@ -835,7 +838,7 @@ class AsyncProcess {
           canRetry = errorsByServer.canRetryMore(numAttempt);
         }
         hConnection.updateCachedLocations(
-            tableName, actions.get(0).getAction().getRow(), throwable, server);
+            tableName, region, actions.get(0).getAction().getRow(), throwable, server);
         failureCount += actions.size();
 
         for (Action<Row> action : actions) {
@@ -989,7 +992,7 @@ class AsyncProcess {
     }
   }
 
-  /** 
+  /**
    * Only used w/useGlobalErrors ctor argument, for HTable backward compat.
    * @return Whether there were any errors in any request since the last time
    *          {@link #waitForAllPreviousOpsAndReset(List)} was called, or AP was created.

Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu Feb  6 02:04:53 2014
@@ -27,11 +27,11 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.
  * keeps a cache of locations and then knows how to re-calibrate after they move.  You need one
  * of these to talk to your HBase cluster. {@link HConnectionManager} manages instances of this
  * class.  See it for how to get one of these.
- * 
+ *
  * <p>This is NOT a connection to a particular server but to ALL servers in the cluster.  Individual
  * connections are managed at a lower level.
  *
@@ -251,7 +251,9 @@ public interface HConnection extends Abo
    * @return HRegionLocation that describes where to find the region in
    * question
    * @throws IOException if a remote or network exception occurs
+   * @deprecated This is no longer a public API
    */
+  @Deprecated
   public HRegionLocation locateRegion(final TableName tableName,
       final byte [] row) throws IOException;
 
@@ -305,11 +307,12 @@ public interface HConnection extends Abo
    * Update the location cache. This is used internally by HBase, in most cases it should not be
    *  used by the client application.
    * @param tableName the table name
+   * @param regionName the regionName
    * @param rowkey the row
    * @param exception the exception if any. Can be null.
    * @param source the previous location
    */
-  void updateCachedLocations(TableName tableName, byte[] rowkey,
+  void updateCachedLocations(TableName tableName, byte[] regionName, byte[] rowkey,
                                     Object exception, ServerName source);
 
   @Deprecated
@@ -345,7 +348,9 @@ public interface HConnection extends Abo
    *          regions from returned list.
    * @return list of region locations for all regions of table
    * @throws IOException
+   * @deprecated This is no longer a public API
    */
+  @Deprecated
   public List<HRegionLocation> locateRegions(final TableName tableName,
       final boolean useCache,
       final boolean offlined) throws IOException;
@@ -388,6 +393,7 @@ public interface HConnection extends Abo
    * @throws IOException if a remote or network exception occurs
    * @deprecated You can pass master flag but nothing special is done.
    */
+  @Deprecated
   AdminService.BlockingInterface getAdmin(final ServerName serverName, boolean getMaster)
       throws IOException;
 
@@ -478,6 +484,7 @@ public interface HConnection extends Abo
    * @throws IOException if a remote or network exception occurs
    * @deprecated This method will be changed from public to package protected.
    */
+  @Deprecated
   int getCurrentNrHRS() throws IOException;
 
   /**



Mime
View raw message