hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1480585 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/master/snapshot/ test/java/org/apache/hadoop/hbase/master/cleaner/ test/java/org/apache/hadoop/hbase/master/snapshot/ t...
Date Thu, 09 May 2013 10:45:21 GMT
Author: mbertozzi
Date: Thu May  9 10:45:20 2013
New Revision: 1480585

URL: http://svn.apache.org/r1480585
Log:
HBASE-8446 Allow parallel snapshot of different tables

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu
May  9 10:45:20 2013
@@ -2665,8 +2665,8 @@ MasterServices, Server {
     try {
       SnapshotDescription snapshot = request.getSnapshot();
       IsRestoreSnapshotDoneResponse.Builder builder = IsRestoreSnapshotDoneResponse.newBuilder();
-      boolean isRestoring = snapshotManager.isRestoringTable(snapshot);
-      builder.setDone(!isRestoring);
+      boolean done = snapshotManager.isRestoreDone(request.getSnapshot());
+      builder.setDone(done);
       return builder.build();
     } catch (IOException e) {
       throw new ServiceException(e);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
Thu May  9 10:45:20 2013
@@ -37,6 +37,11 @@ public interface SnapshotSentinel {
   public boolean isFinished();
 
   /**
+   * @return -1 if the snapshot is in progress, otherwise the completion timestamp.
+   */
+  public long getCompletionTimestamp();
+
+  /**
    * Actively cancel a running snapshot.
    * @param why Reason for cancellation.
    */
@@ -54,4 +59,11 @@ public interface SnapshotSentinel {
    */
   public ForeignException getExceptionIfFailed();
 
+  /**
+   * Rethrow the exception returned by {@link SnapshotSentinel#getExceptionIfFailed}.
+   * If there is no exception this is a no-op.
+   *
+   * @throws ForeignException all exceptions from remote sources are procedure exceptions
+   */
+  public void rethrowExceptionIfFailed() throws ForeignException;
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
Thu May  9 10:45:20 2013
@@ -70,8 +70,7 @@ public class CloneSnapshotHandler extend
 
   public CloneSnapshotHandler(final MasterServices masterServices,
       final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor,
-      final MetricsMaster metricsMaster)
-      throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+      final MetricsMaster metricsMaster) {
     super(masterServices, masterServices.getMasterFileSystem(), hTableDescriptor,
       masterServices.getConfiguration(), null, masterServices);
     this.metricsMaster = metricsMaster;
@@ -155,6 +154,11 @@ public class CloneSnapshotHandler extend
   }
 
   @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
   }
@@ -173,4 +177,9 @@ public class CloneSnapshotHandler extend
   public ForeignException getExceptionIfFailed() {
     return this.monitor.getException();
   }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
Thu May  9 10:45:20 2013
@@ -62,10 +62,9 @@ public class DisabledTableSnapshotHandle
   /**
    * @param snapshot descriptor of the snapshot to take
    * @param masterServices master services provider
-   * @throws IOException on unexpected error
    */
   public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
-      final MasterServices masterServices, final MetricsMaster metricsMaster) throws IOException
{
+      final MasterServices masterServices, final MetricsMaster metricsMaster) {
     super(snapshot, masterServices, metricsMaster);
 
     // setup the timer

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
Thu May  9 10:45:20 2013
@@ -50,7 +50,7 @@ public class EnabledTableSnapshotHandler
   private final ProcedureCoordinator coordinator;
 
   public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
-      final SnapshotManager manager, final MetricsMaster metricsMaster) throws IOException
{
+      final SnapshotManager manager, final MetricsMaster metricsMaster) {
     super(snapshot, master, metricsMaster);
     this.coordinator = manager.getCoordinator();
   }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
Thu May  9 10:45:20 2013
@@ -163,6 +163,11 @@ public class RestoreSnapshotHandler exte
   }
 
   @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
   }
@@ -178,7 +183,13 @@ public class RestoreSnapshotHandler exte
     this.monitor.receive(new ForeignException(masterServices.getServerName().toString(),
ce));
   }
 
+  @Override
   public ForeignException getExceptionIfFailed() {
     return this.monitor.getException();
   }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
Thu May  9 10:45:20 2013
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.snapshot.
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.zookeeper.KeeperException;
@@ -90,6 +91,19 @@ public class SnapshotManager implements 
   /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
   private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
 
+  /**
+   * Wait time before removing a finished sentinel from the in-progress map
+   *
+   * NOTE: This is used as a safety auto cleanup.
+   * The snapshot and restore handlers map entries are removed when a user asks if a snapshot
or
+   * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API
flow.
+   * In case something fails on the client side and the snapshot/restore state is not reclaimed
+   * after a default timeout, the entry is removed from the in-progress map.
+   * At this point, if the user asks for the snapshot/restore status, the result will be
+   * snapshot done if exists or failed if it doesn't exists.
+   */
+  private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
+
   /** Enable or disable snapshot support */
   public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
 
@@ -111,10 +125,11 @@ public class SnapshotManager implements 
   /** Name of the operation to use in the controller */
   public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
 
-  // TODO - enable having multiple snapshots with multiple monitors/threads
-  // this needs to be configuration based when running multiple snapshots is implemented
+  /** Conf key for # of threads used by the SnapshotManager thread pool */
+  private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
+
   /** number of current operations running on the master */
-  private static final int opThreads = 1;
+  private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
 
   private boolean stopped;
   private final long wakeFrequency;
@@ -125,18 +140,21 @@ public class SnapshotManager implements 
   // Is snapshot feature enabled?
   private boolean isSnapshotSupported = false;
 
-  // A reference to a handler.  If the handler is non-null, then it is assumed that a snapshot
is
-  // in progress currently
-  // TODO: this is a bad smell;  likely replace with a collection in the future.  Also this
gets
-  // reset by every operation.
-  private TakeSnapshotHandler handler;
+  // Snapshot handlers map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // snapshotTable() will insert an Handler in the table.
+  // isSnapshotDone() will remove the handler requested if the operation is finished.
+  private Map<String, SnapshotSentinel> snapshotHandlers = new HashMap<String, SnapshotSentinel>();
+
+  // Restore Sentinels map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
+  // isRestoreDone() will remove the handler requested if the operation is finished.
+  private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
 
   private final Path rootDir;
   private final ExecutorService executorService;
 
-  // Restore Sentinels map, with table name as key
-  private Map<String, SnapshotSentinel> restoreHandlers = new HashMap<String, SnapshotSentinel>();
-
   /**
    * Construct a snapshot manager.
    * @param master
@@ -153,6 +171,7 @@ public class SnapshotManager implements 
     Configuration conf = master.getConfiguration();
     this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
     long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
 
     // setup the default procedure coordinator
     String name = master.getServerName().toString();
@@ -194,7 +213,7 @@ public class SnapshotManager implements 
   public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
     return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
   }
-  
+
   /**
    * Gets the list of all completed snapshots.
    * @param snapshotDir snapshot directory
@@ -291,26 +310,8 @@ public class SnapshotManager implements 
   }
 
   /**
-   * Return the handler if it is currently running and has the same snapshot target name.
-   * @param snapshot
-   * @return null if doesn't match, else a live handler.
-   */
-  private synchronized TakeSnapshotHandler getTakeSnapshotHandler(SnapshotDescription snapshot)
{
-    TakeSnapshotHandler h = this.handler;
-    if (h == null) {
-      return null;
-    }
-
-    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
-      // specified snapshot is to the one currently running
-      return null;
-    }
-
-    return h;
-  }
-
-  /**
    * Check if the specified snapshot is done
+   *
    * @param expected
    * @return true if snapshot is ready to be restored, false if it is still being taken.
    * @throws IOException IOException if error from HDFS or RPC
@@ -325,10 +326,20 @@ public class SnapshotManager implements 
 
     String ssString = ClientSnapshotDescriptionUtils.toString(expected);
 
-    // check to see if the sentinel exists
-    TakeSnapshotHandler handler = getTakeSnapshotHandler(expected);
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress snapshots map.
+    SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
     if (handler == null) {
-      // doesn't exist, check if it is already completely done.
+      // If there's no handler in the in-progress map, it means one of the following:
+      //   - someone has already requested the snapshot state
+      //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
+      //   - the snapshot was never requested
+      // In those cases returns to the user the "done state" if the snapshots exists on disk,
+      // otherwise raise an exception saying that the snapshot is not running and doesn't
exist.
       if (!isSnapshotCompleted(expected)) {
         throw new UnknownSnapshotException("Snapshot " + ssString
             + " is not currently running or one of the known completed snapshots.");
@@ -339,7 +350,7 @@ public class SnapshotManager implements 
 
     // pass on any failure we find in the sentinel
     try {
-      handler.rethrowException();
+      handler.rethrowExceptionIfFailed();
     } catch (ForeignException e) {
       // Give some procedure info on an exception.
       String status;
@@ -364,32 +375,19 @@ public class SnapshotManager implements 
   }
 
   /**
-   * Check to see if there are any snapshots in progress currently.  Currently we have a
-   * limitation only allowing a single snapshot attempt at a time.
-   * @return <tt>true</tt> if there any snapshots in progress, <tt>false</tt>
otherwise
-   * @throws SnapshotCreationException if the snapshot failed
-   */
-  synchronized boolean isTakingSnapshot() throws SnapshotCreationException {
-    // TODO later when we handle multiple there would be a map with ssname to handler.
-    return handler != null && !handler.isFinished();
-  }
-
-  /**
    * Check to see if the specified table has a snapshot in progress.  Currently we have a
-   * limitation only allowing a single snapshot attempt at a time.
+   * limitation only allowing a single snapshot per table at a time.
    * @param tableName name of the table being snapshotted.
    * @return <tt>true</tt> if there is a snapshot in progress on the specified
table.
    */
-  private boolean isTakingSnapshot(final String tableName) {
-    if (handler != null && handler.getSnapshot().getTable().equals(tableName)) {
-      return !handler.isFinished();
-    }
-    return false;
+  synchronized boolean isTakingSnapshot(final String tableName) {
+    SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
+    return handler != null && !handler.isFinished();
   }
 
   /**
    * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that
we
-   * aren't already running a snapshot.
+   * aren't already running a snapshot or restore on the requested table.
    * @param snapshot description of the snapshot we want to start
    * @throws HBaseSnapshotException if the filesystem could not be prepared to start the
snapshot
    */
@@ -399,19 +397,21 @@ public class SnapshotManager implements 
     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
 
     // make sure we aren't already running a snapshot
-    if (isTakingSnapshot()) {
+    if (isTakingSnapshot(snapshot.getTable())) {
+      SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
       throw new SnapshotCreationException("Rejected taking "
           + ClientSnapshotDescriptionUtils.toString(snapshot)
           + " because we are already running another snapshot "
-          + ClientSnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+          + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
     }
 
     // make sure we aren't running a restore on the same table
     if (isRestoringTable(snapshot.getTable())) {
+      SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
       throw new SnapshotCreationException("Rejected taking "
           + ClientSnapshotDescriptionUtils.toString(snapshot)
           + " because we are already have a restore in progress on the same snapshot "
-          + ClientSnapshotDescriptionUtils.toString(this.handler.getSnapshot()), snapshot);
+          + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
     }
 
     try {
@@ -433,31 +433,64 @@ public class SnapshotManager implements 
   }
 
   /**
+   * Take a snapshot of a disabled table.
+   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+    // Take the snapshot of the disabled table
+    DisabledTableSnapshotHandler handler =
+        new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
+  /**
    * Take a snapshot of an enabled table.
-   * <p>
-   * The thread limitation on the executorService's thread pool for snapshots ensures the
-   * snapshot won't be started if there is another snapshot already running. Does
-   * <b>not</b> check to see if another snapshot of the same name already exists.
    * @param snapshot description of the snapshot to take.
    * @throws HBaseSnapshotException if the snapshot could not be started
    */
   private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
       throws HBaseSnapshotException {
-    TakeSnapshotHandler handler;
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // Take the snapshot of the enabled table
+    EnabledTableSnapshotHandler handler =
+        new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
+  /**
+   * Take a snapshot using the specified handler.
+   * On failure the snapshot temporary working directory is removed.
+   * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
+   *       snapshot request if the table is busy with another snapshot/restore operation.
+   * @param snapshot the snapshot description
+   * @param handler the snapshot handler
+   */
+  private synchronized void snapshotTable(SnapshotDescription snapshot,
+      final TakeSnapshotHandler handler) throws HBaseSnapshotException {
     try {
-      handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster).prepare();
+      handler.prepare();
       this.executorService.submit(handler);
-      this.handler = handler;
+      this.snapshotHandlers.put(snapshot.getTable(), handler);
     } catch (Exception e) {
       // cleanup the working directory by trying to delete it from the fs.
       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
       try {
         if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true))
{
-          LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:"
-              + ClientSnapshotDescriptionUtils.toString(snapshot));
+          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:"
+
+              ClientSnapshotDescriptionUtils.toString(snapshot));
         }
       } catch (IOException e1) {
-        LOG.warn("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
             ClientSnapshotDescriptionUtils.toString(snapshot));
       }
       // fail the snapshot
@@ -481,6 +514,9 @@ public class SnapshotManager implements 
 
     LOG.debug("No existing snapshot, attempting snapshot...");
 
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
     // check to see if the table exists
     HTableDescriptor desc = null;
     try {
@@ -508,9 +544,6 @@ public class SnapshotManager implements 
       cpHost.preSnapshot(snapshot, desc);
     }
 
-    // setup the snapshot
-    prepareToTakeSnapshot(snapshot);
-
     // if the table is enabled, then have the RS run actually the snapshot work
     AssignmentManager assignmentMgr = master.getAssignmentManager();
     if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
@@ -538,52 +571,21 @@ public class SnapshotManager implements 
   }
 
   /**
-   * Take a snapshot of a disabled table.
-   * <p>
-   * The thread limitation on the executorService's thread pool for snapshots ensures the
-   * snapshot won't be started if there is another snapshot already running. Does
-   * <b>not</b> check to see if another snapshot of the same name already exists.
-   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
-   * @throws HBaseSnapshotException if the snapshot could not be started
-   */
-  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
-      throws HBaseSnapshotException {
-
-    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
-    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
-
-    DisabledTableSnapshotHandler handler;
-    try {
-      handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster).prepare();
-      this.executorService.submit(handler);
-      this.handler = handler;
-    } catch (Exception e) {
-      // cleanup the working directory by trying to delete it from the fs.
-      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-      try {
-        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true))
{
-          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:"
+
-              ClientSnapshotDescriptionUtils.toString(snapshot));
-        }
-      } catch (IOException e1) {
-        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
-            ClientSnapshotDescriptionUtils.toString(snapshot));
-      }
-      // fail the snapshot
-      throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
-    }
-  }
-
-  /**
    * Set the handler for the current snapshot
    * <p>
    * Exposed for TESTING
+   * @param tableName
    * @param handler handler the master should use
    *
    * TODO get rid of this if possible, repackaging, modify tests.
    */
-  public synchronized void setSnapshotHandlerForTesting(TakeSnapshotHandler handler) {
-    this.handler = handler;
+  public synchronized void setSnapshotHandlerForTesting(final String tableName,
+      final SnapshotSentinel handler) {
+    if (handler != null) {
+      this.snapshotHandlers.put(tableName, handler);
+    } else {
+      this.snapshotHandlers.remove(tableName);
+    }
   }
 
   /**
@@ -595,7 +597,9 @@ public class SnapshotManager implements 
 
   /**
    * Check to see if the snapshot is one of the currently completed snapshots
-   * @param expected snapshot to check
+   * Returns true if the snapshot exists in the "completed snapshots folder".
+   *
+   * @param snapshot expected snapshot to check
    * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem},
<tt>false</tt> if is
    *         not stored
    * @throws IOException if the filesystem throws an unexpected exception,
@@ -619,7 +623,6 @@ public class SnapshotManager implements 
    *
    * @param snapshot Snapshot Descriptor
    * @param hTableDescriptor Table Descriptor of the table to create
-   * @param waitTime timeout before considering the clone failed
    */
   synchronized void cloneSnapshot(final SnapshotDescription snapshot,
       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
@@ -639,7 +642,7 @@ public class SnapshotManager implements 
       CloneSnapshotHandler handler =
         new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster).prepare();
       this.executorService.submit(handler);
-      restoreHandlers.put(tableName, handler);
+      this.restoreHandlers.put(tableName, handler);
     } catch (Exception e) {
       String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
+
         " on table=" + tableName;
@@ -669,8 +672,8 @@ public class SnapshotManager implements 
     HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
     String tableName = reqSnapshot.getTable();
 
-    // stop tracking completed restores
-    cleanupRestoreSentinels();
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
 
     // Execute the restore/clone operation
     if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
@@ -710,14 +713,11 @@ public class SnapshotManager implements 
    *
    * @param snapshot Snapshot Descriptor
    * @param hTableDescriptor Table Descriptor
-   * @param waitTime timeout before considering the restore failed
    */
   private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
       final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
     String tableName = hTableDescriptor.getNameAsString();
 
-    // TODO: There is definite race condition for managing the single handler. We should
fix
-    // and remove the limitation of single snapshot / restore at a time.
     // make sure we aren't running a snapshot on the same table
     if (isTakingSnapshot(tableName)) {
       throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
@@ -748,79 +748,107 @@ public class SnapshotManager implements 
    * @param tableName table under restore
    * @return <tt>true</tt> if there is a restore in progress of the specified
table.
    */
-  private boolean isRestoringTable(final String tableName) {
-    SnapshotSentinel sentinel = restoreHandlers.get(tableName);
+  private synchronized boolean isRestoringTable(final String tableName) {
+    SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
     return(sentinel != null && !sentinel.isFinished());
   }
 
   /**
-   * Returns status of a restore request, specifically comparing source snapshot and target
table
-   * names.  Throws exception if not a known snapshot.
+   * Returns the status of a restore operation.
+   * If the in-progress restore is failed throws the exception that caused the failure.
+   *
    * @param snapshot
-   * @return true if in progress, false if snapshot is completed.
-   * @throws UnknownSnapshotException if specified source snapshot does not exit.
-   * @throws IOException if there was some sort of IO failure
+   * @return false if in progress, true if restore is completed or not requested.
+   * @throws IOException if there was a failure during the restore
    */
-  public boolean isRestoringTable(final SnapshotDescription snapshot) throws IOException
{
-    // check to see if the snapshot is already on the fs
-    if (!isSnapshotCompleted(snapshot)) {
-      throw new UnknownSnapshotException("Snapshot:" + snapshot.getName()
-          + " is not one of the known completed snapshots.");
-    }
+  public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress restore map.
+    SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
 
-    SnapshotSentinel sentinel = getRestoreSnapshotSentinel(snapshot.getTable());
     if (sentinel == null) {
       // there is no sentinel so restore is not in progress.
-      return false;
-    }
-    if (!sentinel.getSnapshot().getName().equals(snapshot.getName())) {
-      // another handler is trying to restore to the table, but it isn't the same snapshot
source.
-      return false;
+      return true;
     }
 
     LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
         + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
-    ForeignException e = sentinel.getExceptionIfFailed();
-    if (e != null) throw e;
+
+    // If the restore is failed, rethrow the exception
+    sentinel.rethrowExceptionIfFailed();
 
     // check to see if we are done
     if (sentinel.isFinished()) {
       LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
           " has completed. Notifying the client.");
-      return false;
+      return true;
     }
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
           ClientSnapshotDescriptionUtils.toString(snapshot));
     }
-    return true;
+    return false;
   }
 
   /**
-   * Get the restore snapshot sentinel for the specified table
-   * @param tableName table under restore
-   * @return the restore snapshot handler
+   * Return the handler if it is currently live and has the same snapshot target name.
+   * The handler is removed from the sentinels map if completed.
+   * @param sentinels live handlers
+   * @param snapshot snapshot description
+   * @return null if doesn't match, else a live handler.
    */
-  private synchronized SnapshotSentinel getRestoreSnapshotSentinel(final String tableName)
{
-    try {
-      return restoreHandlers.get(tableName);
-    } finally {
-      cleanupRestoreSentinels();
+  private synchronized SnapshotSentinel removeSentinelIfFinished(
+      final Map<String, SnapshotSentinel> sentinels, final SnapshotDescription snapshot)
{
+    SnapshotSentinel h = sentinels.get(snapshot.getTable());
+    if (h == null) {
+      return null;
+    }
+
+    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+      // specified snapshot is to the one currently running
+      return null;
     }
+
+    // Remove from the "in-progress" list once completed
+    if (h.isFinished()) {
+      sentinels.remove(snapshot.getTable());
+    }
+
+    return h;
   }
 
   /**
-   * Scan the restore handlers and remove the finished ones.
+   * Removes "abandoned" snapshot/restore requests.
+   * As part of the HBaseAdmin snapshot/restore API the operation status is checked until
completed,
+   * and the in-progress maps are cleaned up when the status of a completed task is requested.
+   * To avoid having sentinels staying around for long time if something client side is failed,
+   * each operation tries to clean up the in-progress maps sentinels finished from a long
time.
    */
-  private synchronized void cleanupRestoreSentinels() {
-    Iterator<Map.Entry<String, SnapshotSentinel>> it = restoreHandlers.entrySet().iterator();
+  private void cleanupSentinels() {
+    cleanupSentinels(this.snapshotHandlers);
+    cleanupSentinels(this.restoreHandlers);
+  }
+
+  /**
+   * Remove the sentinels that are marked as finished and the completion time
+   * has exceeded the removal timeout.
+   * @param sentinels map of sentinels to clean
+   */
+  private synchronized void cleanupSentinels(final Map<String, SnapshotSentinel> sentinels)
{
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    Iterator<Map.Entry<String, SnapshotSentinel>> it = sentinels.entrySet().iterator();
     while (it.hasNext()) {
-        Map.Entry<String, SnapshotSentinel> entry = it.next();
-        SnapshotSentinel sentinel = entry.getValue();
-        if (sentinel.isFinished()) {
-          it.remove();
-        }
+      Map.Entry<String, SnapshotSentinel> entry = it.next();
+      SnapshotSentinel sentinel = entry.getValue();
+      if (sentinel.isFinished() &&
+          (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
+      {
+        it.remove();
+      }
     }
   }
 
@@ -835,7 +863,9 @@ public class SnapshotManager implements 
     // make sure we get stop
     this.stopped = true;
     // pass the stop onto take snapshot handlers
-    if (this.handler != null) this.handler.cancel(why);
+    for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
+      snapshotHandler.cancel(why);
+    }
 
     // pass the stop onto all the restore handlers
     for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
@@ -895,7 +925,7 @@ public class SnapshotManager implements 
       LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
       LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
     }
-    
+
     // If the user has enabled the snapshot, we force the cleaners to be present
     // otherwise we still need to check if cleaners are enabled or not and verify
     // that there're no snapshot in the .snapshot folder.

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
Thu May  9 10:45:20 2013
@@ -87,10 +87,9 @@ public abstract class TakeSnapshotHandle
   /**
    * @param snapshot descriptor of the snapshot to take
    * @param masterServices master services provider
-   * @throws IOException on unexpected error
    */
   public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
-      final MetricsMaster metricsMaster) throws IOException {
+      final MetricsMaster metricsMaster) {
     super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
     assert snapshot != null : "SnapshotDescription must not be nul1";
     assert masterServices != null : "MasterServices must not be nul1";
@@ -264,6 +263,11 @@ public abstract class TakeSnapshotHandle
   }
 
   @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
   public SnapshotDescription getSnapshot() {
     return snapshot;
   }
@@ -274,6 +278,11 @@ public abstract class TakeSnapshotHandle
   }
 
   @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  @Override
   public void rethrowException() throws ForeignException {
     monitor.rethrowException();
   }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
Thu May  9 10:45:20 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.snapshot.
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.exceptions.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -129,7 +130,7 @@ public class TestSnapshotFromMaster {
   @Before
   public void setup() throws Exception {
     UTIL.createTable(TABLE_NAME, TEST_FAM);
-    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
+    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(STRING_TABLE_NAME,
null);
   }
 
   @After
@@ -182,7 +183,8 @@ public class TestSnapshotFromMaster {
       UnknownSnapshotException.class);
 
     // and that we get the same issue, even if we specify a name
-    SnapshotDescription desc = SnapshotDescription.newBuilder().setName(snapshotName).build();
+    SnapshotDescription desc = SnapshotDescription.newBuilder()
+      .setName(snapshotName).setTable(STRING_TABLE_NAME).build();
     builder.setSnapshot(desc);
     SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(),
       UnknownSnapshotException.class);
@@ -192,8 +194,11 @@ public class TestSnapshotFromMaster {
     Mockito.when(mockHandler.getException()).thenReturn(null);
     Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
     Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+    Mockito.when(mockHandler.getCompletionTimestamp())
+      .thenReturn(EnvironmentEdgeManager.currentTimeMillis());
 
-    master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
+    master.getSnapshotManagerForTesting()
+        .setSnapshotHandlerForTesting(STRING_TABLE_NAME, mockHandler);
 
     // if we do a lookup without a snapshot name, we should fail - you should always know
your name
     builder = IsSnapshotDoneRequest.newBuilder();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
Thu May  9 10:45:20 2013
@@ -67,7 +67,8 @@ public class TestSnapshotManager {
     return getNewManager(UTIL.getConfiguration());
   }
 
-  private SnapshotManager getNewManager(final Configuration conf) throws IOException, KeeperException
{
+  private SnapshotManager getNewManager(final Configuration conf)
+      throws IOException, KeeperException {
     Mockito.reset(services);
     Mockito.when(services.getConfiguration()).thenReturn(conf);
     Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
@@ -78,14 +79,18 @@ public class TestSnapshotManager {
 
   @Test
   public void testInProcess() throws KeeperException, IOException {
+    String tableName = "testTable";
     SnapshotManager manager = getNewManager();
     TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class);
-    assertFalse("Manager is in process when there is no current handler", manager.isTakingSnapshot());
-    manager.setSnapshotHandlerForTesting(handler);
+    assertFalse("Manager is in process when there is no current handler",
+        manager.isTakingSnapshot(tableName));
+    manager.setSnapshotHandlerForTesting(tableName, handler);
     Mockito.when(handler.isFinished()).thenReturn(false);
-    assertTrue("Manager isn't in process when handler is running", manager.isTakingSnapshot());
+    assertTrue("Manager isn't in process when handler is running",
+        manager.isTakingSnapshot(tableName));
     Mockito.when(handler.isFinished()).thenReturn(true);
-    assertFalse("Manager is process when handler isn't running", manager.isTakingSnapshot());
+    assertFalse("Manager is process when handler isn't running",
+        manager.isTakingSnapshot(tableName));
   }
 
   /**

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1480585&r1=1480584&r2=1480585&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
Thu May  9 10:45:20 2013
@@ -260,11 +260,8 @@ public class TestFlushSnapshotFromClient
     // load the table so we have some data
     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
     // and wait until everything stabilizes
-    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
-    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
-    for (HRegion region : onlineRegions) {
-      region.waitForFlushesAndCompactions();
-    }
+    waitForTableToBeOnline(TABLE_NAME);
+
     String snapshotName = "flushSnapshotCreateListDestroy";
     // test creating the snapshot
     admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
@@ -314,32 +311,27 @@ public class TestFlushSnapshotFromClient
   }
 
   /**
-   * Demonstrate that we reject snapshot requests if there is a snapshot currently running.
+   * Demonstrate that we reject snapshot requests if there is a snapshot already running
on the
+   * same table currently running and that concurrent snapshots on different tables can both
+   * succeed concurretly.
    */
   @Test(timeout=60000)
   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException
{
-    int ssNum = 10;
+    final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
+    final byte[] TABLE2_NAME = Bytes.toBytes(STRING_TABLE2_NAME);
+
+    int ssNum = 20;
     HBaseAdmin admin = UTIL.getHBaseAdmin();
     // make sure we don't fail on listing snapshots
     SnapshotTestingUtils.assertNoSnapshots(admin);
+    // create second testing table
+    UTIL.createTable(TABLE2_NAME, TEST_FAM);
     // load the table so we have some data
     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
+    UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE2_NAME), TEST_FAM);
     // and wait until everything stabilizes
-    HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
-    List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
-    for (HRegion region : onlineRegions) {
-      region.waitForFlushesAndCompactions();
-    }
-
-    // build descriptions
-    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
-    for (int i = 0; i < ssNum; i++) {
-      SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
-      builder.setTable(STRING_TABLE_NAME);
-      builder.setName("ss"+i);
-      builder.setType(SnapshotDescription.Type.FLUSH);
-      descs[i] = builder.build();
-    }
+    waitForTableToBeOnline(TABLE_NAME);
+    waitForTableToBeOnline(TABLE2_NAME);
 
     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
     // We'll have one of these per thread
@@ -365,6 +357,16 @@ public class TestFlushSnapshotFromClient
       }
     };
 
+    // build descriptions
+    SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+    for (int i = 0; i < ssNum; i++) {
+      SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+      builder.setTable((i % 2) == 0 ? STRING_TABLE_NAME : STRING_TABLE2_NAME);
+      builder.setName("ss"+i);
+      builder.setType(SnapshotDescription.Type.FLUSH);
+      descs[i] = builder.build();
+    }
+
     // kick each off its own thread
     for (int i=0 ; i < ssNum; i++) {
       new Thread(new SSRunnable(descs[i])).start();
@@ -400,13 +402,36 @@ public class TestFlushSnapshotFromClient
     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
         " issued many requests", takenSize < ssNum && takenSize > 0);
+
+    // Verify that there's at least one snapshot per table
+    int t1SnapshotsCount = 0;
+    int t2SnapshotsCount = 0;
+    for (SnapshotDescription ss : taken) {
+      if (ss.getTable().equals(STRING_TABLE_NAME)) {
+        t1SnapshotsCount++;
+      } else if (ss.getTable().equals(STRING_TABLE2_NAME)) {
+        t2SnapshotsCount++;
+      }
+    }
+    assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
+    assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
+
     // delete snapshots so subsequent tests are clean.
     for (SnapshotDescription ss : taken) {
       admin.deleteSnapshot(ss.getName());
     }
+    UTIL.deleteTable(TABLE2_NAME);
   }
 
   private void logFSTree(Path root) throws IOException {
     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
   }
+
+  private void waitForTableToBeOnline(final byte[] tableName) throws IOException {
+    HRegionServer rs = UTIL.getRSForFirstRegionInTable(tableName);
+    List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+    for (HRegion region : onlineRegions) {
+      region.waitForFlushesAndCompactions();
+    }
+  }
 }



Mime
View raw message