hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [5/5] hbase git commit: HBASE-15521 Procedure V2 - RestoreSnapshot and CloneSnapshot (Stephen Yuan Jiang)
Date Fri, 01 Apr 2016 04:49:40 GMT
HBASE-15521 Procedure V2 - RestoreSnapshot and CloneSnapshot (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e1d5c3d2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e1d5c3d2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e1d5c3d2

Branch: refs/heads/master
Commit: e1d5c3d269129cf8c313ba29d9a9f694b799170b
Parents: ff6a339
Author: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
Authored: Thu Mar 31 21:49:13 2016 -0700
Committer: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
Committed: Thu Mar 31 21:49:13 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/client/Admin.java   |   36 +-
 .../hbase/client/ConnectionImplementation.java  |    7 -
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |  199 +-
 .../hbase/procedure2/ProcedureExecutor.java     |    2 +-
 .../generated/MasterProcedureProtos.java        | 5804 +++++++++++++++++-
 .../hbase/protobuf/generated/MasterProtos.java  | 1040 ++--
 hbase-protocol/src/main/protobuf/Master.proto   |    8 +-
 .../src/main/protobuf/MasterProcedure.proto     |   40 +
 .../hadoop/hbase/master/MasterRpcServices.java  |   32 +-
 .../procedure/CloneSnapshotProcedure.java       |  522 ++
 .../master/procedure/CreateTableProcedure.java  |   16 +-
 .../procedure/MasterDDLOperationHelper.java     |   13 +
 .../procedure/RestoreSnapshotProcedure.java     |  526 ++
 .../master/snapshot/CloneSnapshotHandler.java   |  195 -
 .../master/snapshot/RestoreSnapshotHandler.java |  245 -
 .../hbase/master/snapshot/SnapshotManager.java  |  330 +-
 .../hbase/snapshot/RestoreSnapshotHelper.java   |   14 +-
 .../client/TestSnapshotCloneIndependence.java   |    1 -
 .../MasterProcedureTestingUtility.java          |    4 +-
 .../procedure/TestCloneSnapshotProcedure.java   |  239 +
 .../procedure/TestCreateTableProcedure.java     |    4 +-
 .../procedure/TestDeleteTableProcedure.java     |    8 +-
 .../TestMasterFailoverWithProcedures.java       |    2 +-
 .../master/procedure/TestProcedureAdmin.java    |    2 +-
 .../procedure/TestRestoreSnapshotProcedure.java |  291 +
 25 files changed, 8200 insertions(+), 1380 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e1d5c3d2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index c3b524b..54d2cb9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1330,6 +1330,22 @@ public interface Admin extends Abortable, Closeable {
   void restoreSnapshot(final String snapshotName) throws IOException, RestoreSnapshotException;
 
   /**
+   * Restore the specified snapshot on the original table. (The table must be disabled) If
the
+   * "hbase.snapshot.restore.take.failsafe.snapshot" configuration property is set to true,
a
+   * snapshot of the current table is taken before executing the restore operation. In case
of
+   * restore failure, the failsafe snapshot will be restored. If the restore completes without
+   * problem the failsafe snapshot is deleted.
+   *
+   * @param snapshotName name of the snapshot to restore
+   * @throws IOException if a remote or network exception occurs
+   * @throws RestoreSnapshotException if snapshot failed to be restored
+   * @return the result of the async restore snapshot. You can use Future.get(long, TimeUnit)
+   *    to wait on the operation to complete.
+   */
+  Future<Void> restoreSnapshotAsync(final String snapshotName)
+      throws IOException, RestoreSnapshotException;
+
+  /**
    * Restore the specified snapshot on the original table. (The table must be disabled) If
    * 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken before
    * executing the restore operation. In case of restore failure, the failsafe snapshot will
be
@@ -1360,7 +1376,7 @@ public interface Admin extends Abortable, Closeable {
    * @throws RestoreSnapshotException if snapshot failed to be restored
    * @throws IllegalArgumentException if the restore request is formatted incorrectly
    */
-  void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
+  void restoreSnapshot(final String snapshotName, final boolean takeFailSafeSnapshot)
       throws IOException, RestoreSnapshotException;
 
   /**
@@ -1390,6 +1406,24 @@ public interface Admin extends Abortable, Closeable {
       throws IOException, TableExistsException, RestoreSnapshotException;
 
   /**
+   * Create a new table by cloning the snapshot content, but does not block
+   * and wait for it be completely cloned.
+   * You can use Future.get(long, TimeUnit) to wait on the operation to complete.
+   * It may throw ExecutionException if there was an error while executing the operation
+   * or TimeoutException in case the wait timeout was not long enough to allow the
+   * operation to complete.
+   *
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @throws IOException if a remote or network exception occurs
+   * @throws TableExistsException if table to be cloned already exists
+   * @return the result of the async clone snapshot. You can use Future.get(long, TimeUnit)
+   *    to wait on the operation to complete.
+   */
+  Future<Void> cloneSnapshotAsync(final String snapshotName, final TableName tableName)
+      throws IOException, TableExistsException;
+
+  /**
    * Execute a distributed procedure on a cluster.
    *
    * @param signature A distributed procedure is uniquely identified by its signature (default
the

http://git-wip-us.apache.org/repos/asf/hbase/blob/e1d5c3d2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index ecaf18b..c71d81a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -1588,13 +1588,6 @@ class ConnectionImplementation implements ClusterConnection, Closeable
{
       }
 
       @Override
-      public MasterProtos.IsRestoreSnapshotDoneResponse isRestoreSnapshotDone(
-          RpcController controller, MasterProtos.IsRestoreSnapshotDoneRequest request)
-          throws ServiceException {
-        return stub.isRestoreSnapshotDone(controller, request);
-      }
-
-      @Override
       public MasterProtos.ExecProcedureResponse execProcedure(
           RpcController controller, MasterProtos.ExecProcedureRequest request)
           throws ServiceException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/e1d5c3d2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index c1d07ae..aea86b9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -127,8 +127,6 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescripto
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
@@ -693,47 +691,6 @@ public class HBaseAdmin implements Admin {
     get(enableTableAsync(tableName), syncWaitTimeout, TimeUnit.MILLISECONDS);
   }
 
-  /**
-   * Wait for the table to be enabled and available
-   * If enabling the table exceeds the retry period, an exception is thrown.
-   * @param tableName name of the table
-   * @throws IOException if a remote or network exception occurs or
-   *    table is not enabled after the retries period.
-   */
-  private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
-    boolean enabled = false;
-    long start = EnvironmentEdgeManager.currentTime();
-    for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++)
{
-      try {
-        enabled = isTableEnabled(tableName);
-      } catch (TableNotFoundException tnfe) {
-        // wait for table to be created
-        enabled = false;
-      }
-      enabled = enabled && isTableAvailable(tableName);
-      if (enabled) {
-        break;
-      }
-      long sleep = getPauseTime(tries);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
-          "enabled in " + tableName);
-      }
-      try {
-        Thread.sleep(sleep);
-      } catch (InterruptedException e) {
-        // Do this conversion rather than let it out because do not want to
-        // change the method signature.
-        throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
-      }
-    }
-    if (!enabled) {
-      long msec = EnvironmentEdgeManager.currentTime() - start;
-      throw new IOException("Table '" + tableName +
-        "' not yet enabled, after " + msec + "ms.");
-    }
-  }
-
   @Override
   public Future<Void> enableTableAsync(final TableName tableName) throws IOException
{
     TableName.isLegalFullyQualifiedTableName(tableName.getName());
@@ -2430,8 +2387,14 @@ public class HBaseAdmin implements Admin {
     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
   }
 
-  @Override
-  public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
+  /*
+   * Check whether the snapshot exists and contains disabled table
+   *
+   * @param snapshotName name of the snapshot to restore
+   * @throws IOException if a remote or network exception occurs
+   * @throws RestoreSnapshotException if no valid snapshot is found
+   */
+  private TableName getTableNameBeforeRestoreSnapshot(final String snapshotName)
       throws IOException, RestoreSnapshotException {
     TableName tableName = null;
     for (SnapshotDescription snapshotInfo: listSnapshots()) {
@@ -2445,6 +2408,13 @@ public class HBaseAdmin implements Admin {
       throw new RestoreSnapshotException(
         "Unable to find the table name for snapshot=" + snapshotName);
     }
+    return tableName;
+  }
+
+  @Override
+  public void restoreSnapshot(final String snapshotName, final boolean takeFailSafeSnapshot)
+      throws IOException, RestoreSnapshotException {
+    TableName tableName = getTableNameBeforeRestoreSnapshot(snapshotName);
 
     // The table does not exists, switch to clone.
     if (!tableExists(tableName)) {
@@ -2472,13 +2442,19 @@ public class HBaseAdmin implements Admin {
 
     try {
       // Restore snapshot
-      internalRestoreSnapshot(snapshotName, tableName);
+      get(
+        internalRestoreSnapshotAsync(snapshotName, tableName),
+        syncWaitTimeout,
+        TimeUnit.MILLISECONDS);
     } catch (IOException e) {
       // Somthing went wrong during the restore...
       // if the pre-restore snapshot is available try to rollback
       if (takeFailSafeSnapshot) {
         try {
-          internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
+          get(
+            internalRestoreSnapshotAsync(failSafeSnapshotSnapshotName, tableName),
+            syncWaitTimeout,
+            TimeUnit.MILLISECONDS);
           String msg = "Restore snapshot=" + snapshotName +
             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
           LOG.error(msg, e);
@@ -2505,6 +2481,24 @@ public class HBaseAdmin implements Admin {
   }
 
   @Override
+  public Future<Void> restoreSnapshotAsync(final String snapshotName)
+      throws IOException, RestoreSnapshotException {
+    TableName tableName = getTableNameBeforeRestoreSnapshot(snapshotName);
+
+    // The table does not exists, switch to clone.
+    if (!tableExists(tableName)) {
+      return cloneSnapshotAsync(snapshotName, tableName);
+    }
+
+    // Check if the table is disabled
+    if (!isTableDisabled(tableName)) {
+      throw new TableNotDisabledException(tableName);
+    }
+
+    return internalRestoreSnapshotAsync(snapshotName, tableName);
+  }
+
+  @Override
   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
       throws IOException, TableExistsException, RestoreSnapshotException {
     cloneSnapshot(Bytes.toString(snapshotName), tableName);
@@ -2516,8 +2510,19 @@ public class HBaseAdmin implements Admin {
     if (tableExists(tableName)) {
       throw new TableExistsException(tableName);
     }
-    internalRestoreSnapshot(snapshotName, tableName);
-    waitUntilTableIsEnabled(tableName);
+    get(
+      internalRestoreSnapshotAsync(snapshotName, tableName),
+      Integer.MAX_VALUE,
+      TimeUnit.MILLISECONDS);
+  }
+
+  @Override
+  public Future<Void> cloneSnapshotAsync(final String snapshotName, final TableName
tableName)
+      throws IOException, TableExistsException {
+    if (tableExists(tableName)) {
+      throw new TableExistsException(tableName);
+    }
+    return internalRestoreSnapshotAsync(snapshotName, tableName);
   }
 
   @Override
@@ -2632,73 +2637,59 @@ public class HBaseAdmin implements Admin {
    * @throws RestoreSnapshotException if snapshot failed to be restored
    * @throws IllegalArgumentException if the restore request is formatted incorrectly
    */
-  private void internalRestoreSnapshot(final String snapshotName, final TableName tableName)
-      throws IOException, RestoreSnapshotException {
-    SnapshotDescription snapshot = SnapshotDescription.newBuilder()
+  private Future<Void> internalRestoreSnapshotAsync(
+      final String snapshotName,
+      final TableName tableName) throws IOException, RestoreSnapshotException {
+    final SnapshotDescription snapshot = SnapshotDescription.newBuilder()
         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
 
     // actually restore the snapshot
-    internalRestoreSnapshotAsync(snapshot);
-
-    final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
-        .setSnapshot(snapshot).build();
-    IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
-        .setDone(false).buildPartial();
-    final long maxPauseTime = 5000;
-    int tries = 0;
-    while (!done.getDone()) {
-      try {
-        // sleep a backoff <= pauseTime amount
-        long sleep = getPauseTime(tries++);
-        sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
-        LOG.debug(tries + ") Sleeping: " + sleep
-            + " ms while we wait for snapshot restore to complete.");
-        Thread.sleep(sleep);
-      } catch (InterruptedException e) {
-        throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
-      }
-      LOG.debug("Getting current status of snapshot restore from master...");
-      done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
-          getConnection()) {
-        @Override
-        public IsRestoreSnapshotDoneResponse call(int callTimeout) throws ServiceException
{
-          PayloadCarryingRpcController controller = rpcControllerFactory.newController();
-          controller.setCallTimeout(callTimeout);
-          return master.isRestoreSnapshotDone(controller, request);
-        }
-      });
-    }
-    if (!done.getDone()) {
-      throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
-    }
-  }
-
-  /**
-   * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
-   * <p>
-   * Only a single snapshot should be restored at a time, or results may be undefined.
-   * @param snapshot snapshot to restore
-   * @return response from the server indicating the max time to wait for the snapshot
-   * @throws IOException if a remote or network exception occurs
-   * @throws RestoreSnapshotException if snapshot failed to be restored
-   * @throws IllegalArgumentException if the restore request is formatted incorrectly
-   */
-  private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription
snapshot)
-      throws IOException, RestoreSnapshotException {
     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
 
-    final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
-        .build();
-
-    // run the snapshot restore on the master
-    return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection())
{
+    RestoreSnapshotResponse response = executeCallable(
+        new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
       @Override
       public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
+        final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder()
+            .setSnapshot(snapshot)
+            .setNonceGroup(ng.getNonceGroup())
+            .setNonce(ng.newNonce())
+            .build();
         PayloadCarryingRpcController controller = rpcControllerFactory.newController();
         controller.setCallTimeout(callTimeout);
         return master.restoreSnapshot(controller, request);
       }
     });
+
+    return new RestoreSnapshotFuture(
+      this, snapshot, TableName.valueOf(snapshot.getTable()), response);
+  }
+
+  private static class RestoreSnapshotFuture extends TableFuture<Void> {
+    public RestoreSnapshotFuture(
+        final HBaseAdmin admin,
+        final SnapshotDescription snapshot,
+        final TableName tableName,
+        final RestoreSnapshotResponse response) {
+      super(admin, tableName,
+          (response != null && response.hasProcId()) ? response.getProcId() : null);
+
+      if (response != null && !response.hasProcId()) {
+        throw new UnsupportedOperationException("Client could not call old version of Server");
+      }
+    }
+
+    public RestoreSnapshotFuture(
+        final HBaseAdmin admin,
+        final TableName tableName,
+        final Long procId) {
+      super(admin, tableName, procId);
+    }
+
+    @Override
+    public String getOperationType() {
+      return "MODIFY";
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/e1d5c3d2/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 38dd062..3f0ba37 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -694,7 +694,7 @@ public class ProcedureExecutor<TEnvironment> {
    * @return true if the procedure execution is finished, otherwise false.
    */
   public boolean isFinished(final long procId) {
-    return completed.containsKey(procId);
+    return !procedures.containsKey(procId);
   }
 
   /**


Mime
View raw message