hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjus...@apache.org
Subject svn commit: r1460306 [1/5] - in /hbase/trunk: hbase-client/src/main/java/org/apache/hadoop/hbase/ hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ hbase-client/src/main/java/org/apac...
Date Sun, 24 Mar 2013 10:26:22 GMT
Author: zjushch
Date: Sun Mar 24 10:26:21 2013
New Revision: 1460306

URL: http://svn.apache.org/r1460306
Log:
HBASE-7403 Online Merge (Chunhui shen)

Added:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DispatchMergingRegionHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MergedRegionHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
Modified:
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/AdminProtos.java
    hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterAdminProtos.java
    hbase/trunk/hbase-protocol/src/main/protobuf/Admin.proto
    hbase/trunk/hbase-protocol/src/main/protobuf/MasterAdmin.proto
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Sun Mar
24 10:26:21 2013
@@ -1107,4 +1107,27 @@ public class HRegionInfo implements Comp
     return hris;
   }
 
+  /**
+   * Check whether two regions are adjacent
+   * @param regionA
+   * @param regionB
+   * @return true if two regions are adjacent
+   */
+  public static boolean areAdjacent(HRegionInfo regionA, HRegionInfo regionB) {
+    if (regionA == null || regionB == null) {
+      throw new IllegalArgumentException(
+          "Can't check whether adjacent for null region");
+    }
+    HRegionInfo a = regionA;
+    HRegionInfo b = regionB;
+    if (Bytes.compareTo(a.getStartKey(), b.getStartKey()) > 0) {
+      a = regionB;
+      b = regionA;
+    }
+    if (Bytes.compareTo(a.getEndKey(), b.getStartKey()) == 0) {
+      return true;
+    }
+    return false;
+  }
+
 }

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
Sun Mar 24 10:26:21 2013
@@ -234,6 +234,38 @@ public class MetaReader {
   }
 
   /**
+   * Gets the result in META for the specified region.
+   * @param catalogTracker
+   * @param regionName
+   * @return result of the specified region
+   * @throws IOException
+   */
+  public static Result getRegionResult(CatalogTracker catalogTracker,
+      byte[] regionName) throws IOException {
+    Get get = new Get(regionName);
+    get.addFamily(HConstants.CATALOG_FAMILY);
+    return get(getCatalogHTable(catalogTracker), get);
+  }
+
+  /**
+   * Get regions from the merge qualifier of the specified merged region
+   * @return null if it doesn't contain merge qualifier, else two merge regions
+   * @throws IOException
+   */
+  public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
+      CatalogTracker catalogTracker, byte[] regionName) throws IOException {
+    Result result = getRegionResult(catalogTracker, regionName);
+    HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
+        HConstants.MERGEA_QUALIFIER);
+    HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
+        HConstants.MERGEB_QUALIFIER);
+    if (mergeA == null && mergeB == null) {
+      return null;
+    }
+    return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
+ }
+
+  /**
    * Checks if the specified table exists.  Looks at the META table hosted on
    * the specified server.
    * @param catalogTracker

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
Sun Mar 24 10:26:21 2013
@@ -18,8 +18,18 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -78,6 +88,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteSnapshotRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DispatchMergingRegionsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneResponse;
@@ -109,17 +120,8 @@ import org.apache.hadoop.ipc.RemoteExcep
 import org.apache.hadoop.util.StringUtils;
 import org.apache.zookeeper.KeeperException;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.SocketTimeoutException;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Pattern;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
 
 /**
  * Provides an interface to manage HBase database table metadata + general
@@ -1690,6 +1692,38 @@ public class HBaseAdmin implements Abort
   }
 
   /**
+   * Merge two regions. Asynchronous operation.
+   * @param encodedNameOfRegionA encoded name of region a
+   * @param encodedNameOfRegionB encoded name of region b
+   * @param forcible true if do a compulsory merge, otherwise we will only merge
+   *          two adjacent regions
+   * @throws IOException
+   */
+  public void mergeRegions(final byte[] encodedNameOfRegionA,
+      final byte[] encodedNameOfRegionB, final boolean forcible)
+      throws IOException {
+    MasterAdminKeepAliveConnection master = connection
+        .getKeepAliveMasterAdmin();
+    try {
+      DispatchMergingRegionsRequest request = RequestConverter
+          .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
+              encodedNameOfRegionB, forcible);
+      master.dispatchMergingRegions(null, request);
+    } catch (ServiceException se) {
+      IOException ioe = ProtobufUtil.getRemoteException(se);
+      if (ioe instanceof UnknownRegionException) {
+        throw (UnknownRegionException) ioe;
+      }
+      LOG.error("Unexpected exception: " + se
+          + " from calling HMaster.dispatchMergingRegions");
+    } catch (DeserializationException de) {
+      LOG.error("Could not parse destination server name: " + de);
+    } finally {
+      master.close();
+    }
+  }
+
+  /**
    * Split a table or an individual region.
    * Asynchronous operation.
    *

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
Sun Mar 24 10:26:21 2013
@@ -43,6 +43,8 @@ public enum EventType {
   RS_ZK_REGION_SPLITTING    (5, null), // RS has started a region split
   RS_ZK_REGION_SPLIT        (6, ExecutorType.MASTER_SERVER_OPERATIONS),   // RS split has
completed.
   RS_ZK_REGION_FAILED_OPEN  (7, ExecutorType.MASTER_CLOSE_REGION),   // RS failed to open
a region
+  RS_ZK_REGION_MERGING      (8, null), // RS has started merging regions
+  RS_ZK_REGION_MERGE        (9, ExecutorType.MASTER_SERVER_OPERATIONS),   // RS region merge
has completed.
 
   // Messages originating from Master to RS
   M_RS_OPEN_REGION          (20, ExecutorType.RS_OPEN_REGION),  // Master asking RS to open
a region
@@ -53,6 +55,7 @@ public enum EventType {
   M_RS_CLOSE_META           (25, ExecutorType.RS_CLOSE_META),  // Master asking RS to close
meta
 
   // Messages originating from Client to Master
+  C_M_MERGE_REGION          (30, ExecutorType.MASTER_TABLE_OPERATIONS),   // Client asking
Master to merge regions
   C_M_DELETE_TABLE          (40, ExecutorType.MASTER_TABLE_OPERATIONS),   // Client asking
Master to delete a table
   C_M_DISABLE_TABLE         (41, ExecutorType.MASTER_TABLE_OPERATIONS),   // Client asking
Master to disable a table
   C_M_ENABLE_TABLE          (42, ExecutorType.MASTER_TABLE_OPERATIONS),   // Client asking
Master to enable a table

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
Sun Mar 24 10:26:21 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
@@ -1426,6 +1427,28 @@ public final class ProtobufUtil {
     }
   }
 
+  /**
+   * A helper to merge regions using admin protocol. Send request to
+   * regionserver.
+   * @param admin
+   * @param region_a
+   * @param region_b
+   * @param forcible true if do a compulsory merge, otherwise we will only merge
+   *          two adjacent regions
+   * @throws IOException
+   */
+  public static void mergeRegions(final AdminProtocol admin,
+      final HRegionInfo region_a, final HRegionInfo region_b,
+      final boolean forcible) throws IOException {
+    MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
+        region_a.getRegionName(), region_b.getRegionName(),forcible);
+    try {
+      admin.mergeRegions(null, request);
+    } catch (ServiceException se) {
+      throw ProtobufUtil.getRemoteException(se);
+    }
+  }
+
 // End helpers for Admin
 
   /*

Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
(original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
Sun Mar 24 10:26:21 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
@@ -78,6 +79,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DispatchMergingRegionsRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
@@ -777,6 +779,26 @@ public final class RequestConverter {
    return builder.build();
  }
 
+  /**
+   * Create a MergeRegionsRequest for the given regions
+   * @param regionA name of region a
+   * @param regionB name of region b
+   * @param forcible true if it is a compulsory merge
+   * @return a MergeRegionsRequest
+   */
+  public static MergeRegionsRequest buildMergeRegionsRequest(
+      final byte[] regionA, final byte[] regionB, final boolean forcible) {
+    MergeRegionsRequest.Builder builder = MergeRegionsRequest.newBuilder();
+    RegionSpecifier regionASpecifier = buildRegionSpecifier(
+        RegionSpecifierType.REGION_NAME, regionA);
+    RegionSpecifier regionBSpecifier = buildRegionSpecifier(
+        RegionSpecifierType.REGION_NAME, regionB);
+    builder.setRegionA(regionASpecifier);
+    builder.setRegionB(regionBSpecifier);
+    builder.setForcible(forcible);
+    return builder.build();
+  }
+
  /**
   * Create a  CompactRegionRequest for a given region name
   *
@@ -936,6 +958,18 @@ public final class RequestConverter {
     return builder.build();
   }
 
+  public static DispatchMergingRegionsRequest buildDispatchMergingRegionsRequest(
+      final byte[] encodedNameOfRegionA, final byte[] encodedNameOfRegionB,
+      final boolean forcible) throws DeserializationException {
+    DispatchMergingRegionsRequest.Builder builder = DispatchMergingRegionsRequest.newBuilder();
+    builder.setRegionA(buildRegionSpecifier(
+        RegionSpecifierType.ENCODED_REGION_NAME, encodedNameOfRegionA));
+    builder.setRegionB(buildRegionSpecifier(
+        RegionSpecifierType.ENCODED_REGION_NAME, encodedNameOfRegionB));
+    builder.setForcible(forcible);
+    return builder.build();
+  }
+
   /**
    * Create a protocol buffer AssignRegionRequest
    *

Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Sun Mar
24 10:26:21 2013
@@ -381,6 +381,12 @@ public final class HConstants {
   /** The upper-half split region column qualifier */
   public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
 
+  /** The lower-half merge region column qualifier */
+  public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes("mergeA");
+
+  /** The upper-half merge region column qualifier */
+  public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes("mergeB");
+
   /**
    * The meta table version column qualifier.
    * We keep current version of the meta table in this column in <code>-ROOT-</code>



Mime
View raw message