hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1452257 [3/14] - in /hbase/branches/0.94: security/src/main/java/org/apache/hadoop/hbase/security/access/ security/src/test/java/org/apache/hadoop/hbase/security/access/ src/main/jamon/org/apache/hadoop/hbase/tmpl/master/ src/main/java/org...
Date Mon, 04 Mar 2013 11:24:53 GMT
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+
+/**
+ * HFileLink cleaner that determines if a hfile should be deleted.
+ * HFiles can be deleted only if there're no links to them.
+ *
+ * When a HFileLink is created a back reference file is created in:
+ *      /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table
+ * To check if the hfile can be deleted the back references folder must be empty.
+ */
+@InterfaceAudience.Private
+public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class);
+
+  private FileSystem fs = null;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    if (this.fs == null) return false;
+
+    // HFile Link is always deletable
+    if (HFileLink.isHFileLink(filePath)) return true;
+
+    // If the file is inside a link references directory, means that is a back ref link.
+    // The back ref can be deleted only if the referenced file doesn't exists.
+    Path parentDir = filePath.getParent();
+    if (HFileLink.isBackReferencesDir(parentDir)) {
+      try {
+        Path hfilePath = HFileLink.getHFileFromBackReference(getConf(), filePath);
+        return !fs.exists(hfilePath);
+      } catch (IOException e) {
+        LOG.error("Couldn't verify if the referenced file still exists, keep it just in case");
+        return false;
+      }
+    }
+
+    // HFile is deletable only if has no links
+    try {
+      Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
+      return FSUtils.listStatus(fs, backRefDir) == null;
+    } catch (IOException e) {
+      LOG.error("Couldn't get the references, not deleting file, just in case");
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+
+    // setup filesystem
+    try {
+      this.fs = FileSystem.get(this.getConf());
+    } catch (IOException e) {
+      LOG.error("Couldn't instantiate the file system, not deleting file, just in case");
+    }
+  }
+}

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Mon Mar  4 11:24:50 2013
@@ -1,5 +1,4 @@
 /**
- * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -20,13 +19,25 @@
 package org.apache.hadoop.hbase.master.handler;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -41,21 +52,23 @@ import org.apache.hadoop.hbase.master.As
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.zookeeper.KeeperException;
 
 /**
  * Handler to create a table.
  */
+@InterfaceAudience.Private
 public class CreateTableHandler extends EventHandler {
   private static final Log LOG = LogFactory.getLog(CreateTableHandler.class);
-  private MasterFileSystem fileSystemManager;
-  private final HTableDescriptor hTableDescriptor;
-  private Configuration conf;
-  private final AssignmentManager assignmentManager;
-  private final CatalogTracker catalogTracker;
-  private final ServerManager serverManager;
+  protected MasterFileSystem fileSystemManager;
+  protected final HTableDescriptor hTableDescriptor;
+  protected Configuration conf;
+  protected final AssignmentManager assignmentManager;
+  protected final CatalogTracker catalogTracker;
+  protected final ServerManager serverManager;
   private final HRegionInfo [] newRegions;
 
   public CreateTableHandler(Server server, MasterFileSystem fileSystemManager,
@@ -98,8 +111,7 @@ public class CreateTableHandler extends 
     // table in progress. This will introduce a new zookeeper call. Given
     // createTable isn't a frequent operation, that should be ok.
     try {
-      if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(
-        tableName))
+      if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName))
         throw new TableExistsException(tableName);
     } catch (KeeperException e) {
       throw new IOException("Unable to ensure that the table will be" +
@@ -122,66 +134,91 @@ public class CreateTableHandler extends 
   public void process() {
     String tableName = this.hTableDescriptor.getNameAsString();
     try {
-      LOG.info("Attemping to create the table " + tableName);
-      handleCreateTable();
-    } catch (IOException e) {
-      LOG.error("Error trying to create the table " + tableName, e);
-    } catch (KeeperException e) {
+      LOG.info("Attempting to create the table " + tableName);
+      handleCreateTable(tableName);
+      completed(null);
+    } catch (Throwable e) {
       LOG.error("Error trying to create the table " + tableName, e);
+      completed(e);
     }
   }
 
-  private void handleCreateTable() throws IOException, KeeperException {
-
-    // TODO: Currently we make the table descriptor and as side-effect the
-    // tableDir is created.  Should we change below method to be createTable
-    // where we create table in tmp dir with its table descriptor file and then
-    // do rename to move it into place?
-    FSTableDescriptors.createTableDescriptor(this.hTableDescriptor, this.conf);
-
-    List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
-    final int batchSize =
-      this.conf.getInt("hbase.master.createtable.batchsize", 100);
-    for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
-      HRegionInfo newRegion = this.newRegions[regionIdx];
-      // 1. Create HRegion
-      HRegion region = HRegion.createHRegion(newRegion,
-        this.fileSystemManager.getRootDir(), this.conf,
-        this.hTableDescriptor, null, false, true);
-
-      regionInfos.add(region.getRegionInfo());
-      if (regionIdx % batchSize == 0) {
-        // 2. Insert into META
-        MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
-        regionInfos.clear();
-      }
+  /**
+   * Called after that process() is completed.
+   * @param exception null if process() is successful or not null if something has failed.
+   */
+  protected void completed(final Throwable exception) {
+  }
 
-      // 3. Close the new region to flush to disk.  Close log file too.
-      region.close();
+  /**
+   * Responsible of table creation (on-disk and META) and assignment.
+   * - Create the table directory and descriptor (temp folder)
+   * - Create the on-disk regions (temp folder)
+   *   [If something fails here: we've just some trash in temp]
+   * - Move the table from temp to the root directory
+   *   [If something fails here: we've the table in place but some of the rows required
+   *    present in META. (hbck needed)]
+   * - Add regions to META
+   *   [If something fails here: we don't have regions assigned: table disabled]
+   * - Assign regions to Region Servers
+   *   [If something fails here: we still have the table in disabled state]
+   * - Update ZooKeeper with the enabled state
+   */
+  private void handleCreateTable(String tableName) throws IOException, KeeperException {
+    Path tempdir = fileSystemManager.getTempDir();
+    FileSystem fs = fileSystemManager.getFileSystem();
+
+    // 1. Create Table Descriptor
+    FSTableDescriptors.createTableDescriptor(fs, tempdir, this.hTableDescriptor);
+    Path tempTableDir = new Path(tempdir, tableName);
+    Path tableDir = new Path(fileSystemManager.getRootDir(), tableName);
+
+    // 2. Create Regions
+    List<HRegionInfo> regionInfos = handleCreateHdfsRegions(tempdir, tableName);
+
+    // 3. Move Table temp directory to the hbase root location
+    if (!fs.rename(tempTableDir, tableDir)) {
+      throw new IOException("Unable to move table from temp=" + tempTableDir +
+        " to hbase root=" + tableDir);
     }
-    if (regionInfos.size() > 0) {
+
+    if (regionInfos != null && regionInfos.size() > 0) {
+      // 4. Add regions to META
       MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
-    }
 
-    // 4. Trigger immediate assignment of the regions in round-robin fashion
-    List<ServerName> servers = serverManager.getOnlineServersList();
-    // Remove the deadNotExpired servers from the server list.
-    assignmentManager.removeDeadNotExpiredServers(servers);
-    try {
-      this.assignmentManager.assignUserRegions(Arrays.asList(newRegions),
-        servers);
-    } catch (InterruptedException ie) {
-      LOG.error("Caught " + ie + " during round-robin assignment");
-      throw new IOException(ie);
+      // 5. Trigger immediate assignment of the regions in round-robin fashion
+      List<ServerName> servers = serverManager.getOnlineServersList();
+      // Remove the deadNotExpired servers from the server list.
+      assignmentManager.removeDeadNotExpiredServers(servers);
+      try {
+        this.assignmentManager.assignUserRegions(regionInfos, servers);
+      } catch (InterruptedException e) {
+        LOG.error("Caught " + e + " during round-robin assignment");
+        InterruptedIOException ie = new InterruptedIOException(e.getMessage());
+        ie.initCause(e);
+        throw ie;
+      }
     }
 
-    // 5. Set table enabled flag up in zk.
+    // 6. Set table enabled flag up in zk.
     try {
-      assignmentManager.getZKTable().
-        setEnabledTable(this.hTableDescriptor.getNameAsString());
+      assignmentManager.getZKTable().setEnabledTable(tableName);
     } catch (KeeperException e) {
-      throw new IOException("Unable to ensure that the table will be" +
+      throw new IOException("Unable to ensure that " + tableName + " will be" +
         " enabled because of a ZooKeeper issue", e);
     }
   }
-}
\ No newline at end of file
+
+  /**
+   * Create the on-disk structure for the table, and returns the regions info.
+   * @param tableRootDir directory where the table is being created
+   * @param tableName name of the table under construction
+   * @return the list of regions created
+   */
+  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
+    final String tableName)
+      throws IOException {
+    return ModifyRegionUtils.createRegions(conf, tableRootDir,
+        hTableDescriptor, newRegions, null);
+  }
+}

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Mon Mar  4 11:24:50 2013
@@ -24,10 +24,14 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.catalog.MetaEditor;
 import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
@@ -47,6 +51,7 @@ public class DeleteTableHandler extends 
   @Override
   protected void handleTableOperation(List<HRegionInfo> regions)
   throws IOException, KeeperException {
+    // 1. Wait because of region in transition
     AssignmentManager am = this.masterServices.getAssignmentManager();
     long waitTime = server.getConfiguration().
       getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
@@ -63,23 +68,39 @@ public class DeleteTableHandler extends 
           waitTime + "ms) for region to leave region " +
           region.getRegionNameAsString() + " in transitions");
       }
-      LOG.debug("Deleting region " + region.getRegionNameAsString() +
-        " from META and FS");
-      // Remove region from META
-      MetaEditor.deleteRegion(this.server.getCatalogTracker(), region);
-      // Delete region from FS
-      this.masterServices.getMasterFileSystem().deleteRegion(region);
     }
-    // Delete table from FS
-    this.masterServices.getMasterFileSystem().deleteTable(tableName);
-    // Update table descriptor cache
-    this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
 
-    // If entry for this table in zk, and up in AssignmentManager, remove it.
+    // 2. Remove regions from META
+    LOG.debug("Deleting regions from META");
+    MetaEditor.deleteRegions(this.server.getCatalogTracker(), regions);
+
+    // 3. Move the table in /hbase/.tmp
+    LOG.debug("Moving table directory to a temp directory");
+    MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    Path tempTableDir = mfs.moveTableToTemp(tableName);
+
+    try {
+      // 4. Delete regions from FS (temp directory)
+      FileSystem fs = mfs.getFileSystem();
+      for (HRegionInfo hri: regions) {
+        LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
+        HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+            tempTableDir, new Path(tempTableDir, hri.getEncodedName()));
+      }
+
+      // 5. Delete table from FS (temp directory)
+      if (!fs.delete(tempTableDir, true)) {
+        LOG.error("Couldn't delete " + tempTableDir);
+      }
+    } finally {
+      // 6. Update table descriptor cache
+      this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
 
-    am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+      // 7. If entry for this table in zk, and up in AssignmentManager, remove it.
+      am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+    }
   }
-  
+
   @Override
   public String toString() {
     String name = "UnknownServerName";

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java Mon Mar  4 11:24:50 2013
@@ -170,6 +170,7 @@ public class DisableTableHandler extends
       while (!server.isStopped() && remaining > 0) {
         Thread.sleep(waitingTimeForEvents);
         regions = assignmentManager.getRegionsOfTable(tableName);
+        LOG.debug("Disable waiting until done; " + remaining + " ms remaining; " + regions);
         if (regions.isEmpty()) break;
         remaining = timeout - (System.currentTimeMillis() - startTime);
       }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java Mon Mar  4 11:24:50 2013
@@ -160,12 +160,14 @@ public abstract class TableEventHandler 
   }
 
   /**
+   * Gets a TableDescriptor from the masterServices.  Can Throw exceptions.
+   *
    * @return Table descriptor for this table
    * @throws TableExistsException
    * @throws FileNotFoundException
    * @throws IOException
    */
-  HTableDescriptor getTableDescriptor()
+  public HTableDescriptor getTableDescriptor()
   throws FileNotFoundException, IOException {
     final String name = Bytes.toString(tableName);
     HTableDescriptor htd =

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,150 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Handler to Clone a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to create a new table with the same
+ * content of the specified snapshot.
+ */
+@InterfaceAudience.Private
+public class CloneSnapshotHandler extends CreateTableHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(CloneSnapshotHandler.class);
+
+  private final static String NAME = "Master CloneSnapshotHandler";
+
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+
+  private volatile boolean stopped = false;
+
+  public CloneSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+      throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+    super(masterServices, masterServices.getMasterFileSystem(), 
+      masterServices.getServerManager(), hTableDescriptor,
+      masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
+      masterServices.getAssignmentManager());
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+  }
+
+  /**
+   * Create the on-disk regions, using the tableRootDir provided by the CreateTableHandler.
+   * The cloned table will be created in a temp directory, and then the CreateTableHandler
+   * will be responsible to add the regions returned by this method to META and do the assignment.
+   */
+  @Override
+  protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, final String tableName)
+      throws IOException {
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    Path tableDir = new Path(tableRootDir, tableName);
+
+    try {
+      // 1. Execute the on-disk Clone
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // Clone operation should not have stuff to restore or remove
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
+          "A clone should not have regions to restore");
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRemove(),
+          "A clone should not have regions to remove");
+
+      // At this point the clone is complete. Next step is enabling the table.
+      LOG.info("Clone snapshot=" + snapshot.getName() + " on table=" + tableName + " completed!");
+
+      // 2. let the CreateTableHandler add the regions to meta
+      return metaChanges.getRegionsToAdd();
+    } catch (Exception e) {
+      String msg = "clone snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " failed";
+      LOG.error(msg, e);
+      IOException rse = new RestoreSnapshotException(msg, e, snapshot);
+
+      // these handlers aren't futures so we need to register the error here.
+      this.monitor.receive(new ForeignException(NAME, rse));
+      throw rse;
+    }
+  }
+
+  @Override
+  protected void completed(final Throwable exception) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    LOG.info("Stopping clone snapshot=" + snapshot + " because: " + why);
+    this.monitor.receive(new ForeignException(NAME, new CancellationException(why)));
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Take a snapshot of a disabled table.
+ * <p>
+ * Table must exist when taking the snapshot, or results are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
+  private static final Log LOG = LogFactory.getLog(DisabledTableSnapshotHandler.class);
+  private final TimeoutExceptionInjector timeoutInjector;
+
+  /**
+   * @param snapshot descriptor of the snapshot to take
+   * @param masterServices master services provider
+   * @throws IOException on unexpected error
+   */
+  public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
+      final MasterServices masterServices) throws IOException {
+    super(snapshot, masterServices);
+
+    // setup the timer
+    timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+  }
+
+  // TODO consider parallelizing these operations since they are independent. Right now its just
+  // easier to keep them serial though
+  @Override
+  public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException,
+  KeeperException {
+    try {
+      timeoutInjector.start();
+
+      // 1. get all the regions hosting this table.
+
+      // extract each pair to separate lists
+      Set<String> serverNames = new HashSet<String>();
+      Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
+      for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
+        regions.add(p.getFirst());
+        serverNames.add(p.getSecond().toString());
+      }
+
+      // 2. for each region, write all the info to disk
+      LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
+          + SnapshotDescriptionUtils.toString(snapshot));
+      for (HRegionInfo regionInfo : regions) {
+        // 2.1 copy the regionInfo files to the snapshot
+        Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+          regionInfo.getEncodedName());
+        HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+        // check for error for each region
+        monitor.rethrowException();
+
+        // 2.2 for each region, copy over its recovered.edits directory
+        Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+        new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
+        monitor.rethrowException();
+
+        // 2.3 reference all the files in the region
+        new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+        monitor.rethrowException();
+      }
+
+      // 3. write the table info to disk
+      LOG.info("Starting to copy tableinfo for offline snapshot: " +
+      SnapshotDescriptionUtils.toString(snapshot));
+      TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
+          FSUtils.getRootDir(conf));
+      tableInfoCopyTask.call();
+      monitor.rethrowException();
+    } catch (Exception e) {
+      // make sure we capture the exception to propagate back to the client later
+      String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+          + " due to exception:" + e.getMessage();
+      ForeignException ee = new ForeignException(reason, e);
+      monitor.receive(ee);
+    } finally {
+      LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot)
+          + " as finished.");
+
+      // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+      // operation any further
+      timeoutInjector.complete();
+    }
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.util.Pair;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Handle the master side of taking a snapshot of an online table, regardless of snapshot type.
+ * Uses a {@link Procedure} to run the snapshot across all the involved region servers.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Private
+public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
+
+  private static final Log LOG = LogFactory.getLog(EnabledTableSnapshotHandler.class);
+  private final ProcedureCoordinator coordinator;
+
+  public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
+      SnapshotManager manager) throws IOException {
+    super(snapshot, master);
+    this.coordinator = manager.getCoordinator();
+  }
+
+  // TODO consider switching over to using regionnames, rather than server names. This would allow
+  // regions to migrate during a snapshot, and then be involved when they are ready. Still want to
+  // enforce a snapshot time constraints, but lets us be potentially a bit more robust.
+
+  /**
+   * This method kicks off a snapshot procedure.  Other than that it hangs around for various
+   * phases to complete.
+   */
+  @Override
+  protected void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
+      throws HBaseSnapshotException {
+    Set<String> regionServers = new HashSet<String>(regions.size());
+    for (Pair<HRegionInfo, ServerName> region : regions) {
+      regionServers.add(region.getSecond().toString());
+    }
+
+    // start the snapshot on the RS
+    Procedure proc = coordinator.startProcedure(this.monitor, this.snapshot.getName(),
+      this.snapshot.toByteArray(), Lists.newArrayList(regionServers));
+    if (proc == null) {
+      String msg = "Failed to submit distributed procedure for snapshot '"
+          + snapshot.getName() + "'";
+      LOG.error(msg);
+      throw new HBaseSnapshotException(msg);
+    }
+
+    try {
+      // wait for the snapshot to complete.  A timer thread is kicked off that should cancel this
+      // if it takes too long.
+      proc.waitForCompleted();
+      LOG.info("Done waiting - snapshot for " + this.snapshot.getName() + " finished!");
+    } catch (InterruptedException e) {
+      ForeignException ee =
+          new ForeignException("Interrupted while waiting for snapshot to finish", e);
+      monitor.receive(ee);
+      Thread.currentThread().interrupt();
+    } catch (ForeignException e) {
+      monitor.receive(e);
+    }
+  }
+}
\ No newline at end of file

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * General snapshot verification on the master.
+ * <p>
+ * This is a light-weight verification mechanism for all the files in a snapshot. It doesn't
+ * attempt to verify that the files are exact copies (that would be paramount to taking the
+ * snapshot again!), but instead just attempts to ensure that the files match the expected
+ * files and are the same length.
+ * <p>
+ * Taking an online snapshots can race against other operations and this is an last line of
+ * defense.  For example, if meta changes between when snapshots are taken not all regions of a
+ * table may be present.  This can be caused by a region split (daughters present on this scan,
+ * but snapshot took parent), or move (snapshots only checks lists of region servers, a move could
+ * have caused a region to be skipped or done twice).
+ * <p>
+ * Current snapshot files checked:
+ * <ol>
+ * <li>SnapshotDescription is readable</li>
+ * <li>Table info is readable</li>
+ * <li>Regions</li>
+ * <ul>
+ * <li>Matching regions in the snapshot as currently in the table</li>
+ * <li>{@link HRegionInfo} matches the current and stored regions</li>
+ * <li>All referenced hfiles have valid names</li>
+ * <li>All the hfiles are present (either in .archive directory in the region)</li>
+ * <li>All recovered.edits files are present (by name) and have the correct file size</li>
+ * </ul>
+ * </ol>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class MasterSnapshotVerifier {
+
+  private SnapshotDescription snapshot;
+  private FileSystem fs;
+  private Path rootDir;
+  private String tableName;
+  private MasterServices services;
+
+  /**
+   * @param services services for the master
+   * @param snapshot snapshot to check
+   * @param rootDir root directory of the hbase installation.
+   */
+  public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
+    this.fs = services.getMasterFileSystem().getFileSystem();
+    this.services = services;
+    this.snapshot = snapshot;
+    this.rootDir = rootDir;
+    this.tableName = snapshot.getTable();
+  }
+
+  /**
+   * Verify that the snapshot in the directory is a valid snapshot
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot
+   * @throws CorruptedSnapshotException if the snapshot is invalid
+   * @throws IOException if there is an unexpected connection issue to the filesystem
+   */
+  public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    // verify snapshot info matches
+    verifySnapshotDescription(snapshotDir);
+
+    // check that tableinfo is a valid table description
+    verifyTableInfo(snapshotDir);
+
+    // check that each region is valid
+    verifyRegions(snapshotDir);
+  }
+
+  /**
+   * Check that the snapshot description written in the filesystem matches the current snapshot
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
+    SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    if (!this.snapshot.equals(found)) {
+      throw new CorruptedSnapshotException("Snapshot read (" + found
+          + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
+    }
+  }
+
+  /**
+   * Check that the table descriptor for the snapshot is a valid table descriptor
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifyTableInfo(Path snapshotDir) throws IOException {
+    FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+  }
+
+  /**
+   * Check that all the regions in the snapshot are valid, and accounted for.
+   * @param snapshotDir snapshot directory to check
+   * @throws IOException if we can't reach .META. or read the files from the FS
+   */
+  private void verifyRegions(Path snapshotDir) throws IOException {
+    List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
+      Bytes.toBytes(tableName));
+    for (HRegionInfo region : regions) {
+      // if offline split parent, skip it
+      if (region.isOffline() && (region.isSplit() || region.isSplitParent())) {
+        continue;
+      }
+
+      verifyRegion(fs, snapshotDir, region);
+    }
+  }
+
+  /**
+   * Verify that the region (regioninfo, hfiles) are valid
+   * @param fs the FileSystem instance
+   * @param snapshotDir snapshot directory to check
+   * @param region the region to check
+   */
+  private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) throws IOException {
+    // make sure we have region in the snapshot
+    Path regionDir = new Path(snapshotDir, region.getEncodedName());
+    if (!fs.exists(regionDir)) {
+      // could happen due to a move or split race.
+      throw new CorruptedSnapshotException("No region directory found for region:" + region,
+          snapshot);
+    }
+    // make sure we have the region info in the snapshot
+    Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
+    // make sure the file exists
+    if (!fs.exists(regionInfo)) {
+      throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
+    }
+    FSDataInputStream in = fs.open(regionInfo);
+    HRegionInfo found = new HRegionInfo();
+    try {
+      found.readFields(in);
+      if (!region.equals(found)) {
+        throw new CorruptedSnapshotException("Found region info (" + found
+           + ") doesn't match expected region:" + region, snapshot);
+      }
+    } finally {
+      in.close();
+    }
+
+    // make sure we have the expected recovered edits files
+    TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
+
+    // check for the existance of each hfile
+    PathFilter familiesDirs = new FSUtils.FamilyDirFilter(fs);
+    FileStatus[] columnFamilies = FSUtils.listStatus(fs, regionDir, familiesDirs);
+    // should we do some checking here to make sure the cfs are correct?
+    if (columnFamilies == null) return;
+
+    // setup the suffixes for the snapshot directories
+    Path tableNameSuffix = new Path(tableName);
+    Path regionNameSuffix = new Path(tableNameSuffix, region.getEncodedName());
+
+    // get the potential real paths
+    Path archivedRegion = new Path(HFileArchiveUtil.getArchivePath(services.getConfiguration()),
+        regionNameSuffix);
+    Path realRegion = new Path(rootDir, regionNameSuffix);
+
+    // loop through each cf and check we can find each of the hfiles
+    for (FileStatus cf : columnFamilies) {
+      FileStatus[] hfiles = FSUtils.listStatus(fs, cf.getPath(), null);
+      // should we check if there should be hfiles?
+      if (hfiles == null || hfiles.length == 0) continue;
+
+      Path realCfDir = new Path(realRegion, cf.getPath().getName());
+      Path archivedCfDir = new Path(archivedRegion, cf.getPath().getName());
+      for (FileStatus hfile : hfiles) {
+        // make sure the name is correct
+        if (!StoreFile.validateStoreFileName(hfile.getPath().getName())) {
+          throw new CorruptedSnapshotException("HFile: " + hfile.getPath()
+              + " is not a valid hfile name.", snapshot);
+        }
+
+        // check to see if hfile is present in the real table
+        String fileName = hfile.getPath().getName();
+        Path file = new Path(realCfDir, fileName);
+        Path archived = new Path(archivedCfDir, fileName);
+        if (!fs.exists(file) && !file.equals(archived)) {
+          throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
+              + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
+              + ") directory for the primary table.", snapshot);
+        }
+      }
+    }
+  }
+
+  /**
+   * Check that the logs stored in the log directory for the snapshot are valid - it contains all
+   * the expected logs for all servers involved in the snapshot.
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers list of the names of servers involved in the snapshot.
+   * @throws CorruptedSnapshotException if the hlogs in the snapshot are not correct
+   * @throws IOException if we can't reach the filesystem
+   */
+  private void verifyLogs(Path snapshotDir, Set<String> snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+    Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+    TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers, snapshot,
+      snapshotLogDir);
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,155 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handler to Restore a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to replace the table content with the
+ * data available in the snapshot.
+ */
+@InterfaceAudience.Private
+public class RestoreSnapshotHandler extends TableEventHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(RestoreSnapshotHandler.class);
+
+  private final HTableDescriptor hTableDescriptor;
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+  private volatile boolean stopped = false;
+
+  public RestoreSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor htd)
+      throws IOException {
+    super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+
+    // Check table exists.
+    getTableDescriptor();
+
+    // This is the new schema we are going to write out as this modification.
+    this.hTableDescriptor = htd;
+  }
+
+  /**
+   * The restore table is executed in place.
+   *  - The on-disk data will be restored - reference files are put in place without moving data
+   *  -  [if something fail here: you need to delete the table and re-run the restore]
+   *  - META will be updated
+   *  -  [if something fail here: you need to run hbck to fix META entries]
+   * The passed in list gets changed in this method
+   */
+  @Override
+  protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
+    MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem();
+    CatalogTracker catalogTracker = masterServices.getCatalogTracker();
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    byte[] tableName = hTableDescriptor.getName();
+    Path tableDir = HTableDescriptor.getTableDir(rootDir, tableName);
+
+    try {
+      // 1. Update descriptor
+      this.masterServices.getTableDescriptors().add(hTableDescriptor);
+
+      // 2. Execute the on-disk Restore
+      LOG.debug("Starting restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot));
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
+          masterServices.getConfiguration(), fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // 3. Applies changes to .META.
+      hris.clear();
+      if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
+      if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
+      List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
+      MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+
+      // At this point the restore is complete. Next step is enabling the table.
+      LOG.info("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " on table=" +
+        Bytes.toString(tableName) + " completed!");
+    } catch (IOException e) {
+      String msg = "restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+          + " failed. Try re-running the restore command.";
+      LOG.error(msg, e);
+      monitor.receive(new ForeignException(masterServices.getServerName().toString(), e));
+      throw new RestoreSnapshotException(msg, e);
+    } finally {
+      this.stopped = true;
+    }
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    String msg = "Stopping restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+        + " because: " + why;
+    LOG.info(msg);
+    CancellationException ce = new CancellationException(why);
+    this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
+  }
+
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Intelligently keep track of all the files for all the snapshots.
+ * <p>
+ * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
+ * miss the directory modification time is used to ensure that we don't rescan directories that we
+ * already have in cache. We only check the modification times of the snapshot directories
+ * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * <p>
+ * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
+ * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
+ * we will ignore updates to that snapshot's files.
+ * <p>
+ * This is sufficient because each snapshot has its own directory and is added via an atomic rename
+ * <i>once</i>, when the snapshot is created. We don't need to worry about the data in the snapshot
+ * being run.
+ * <p>
+ * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
+ * also removed from the cache.
+ * <p>
+ * A SnapshotFileInspector must be passed when creating <tt>this</tt> to allow extraction
+ * of files under the /hbase/.snapshot/[snapshot name] directory, for each snapshot.
+ * This allows you to only cache files under, for instance, all the logs in the .logs directory or
+ * all the files under all the regions.
+ * <p>
+ * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
+ * snapshots and will attempt to cache files from those snapshots as well.
+ * <p>
+ * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotFileCache implements Stoppable {
+  interface SnapshotFileInspector {
+    /**
+     * Returns a collection of file names needed by the snapshot.
+     * @param snapshotDir {@link Path} to the snapshot directory to scan.
+     * @return the collection of file names needed by the snapshot.
+     */
+    Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
+  }
+
+  private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
+  private volatile boolean stop = false;
+  private final FileSystem fs;
+  private final SnapshotFileInspector fileInspector;
+  private final Path snapshotDir;
+  private final Set<String> cache = new HashSet<String>();
+  /**
+   * This is a helper map of information about the snapshot directories so we don't need to rescan
+   * them if they haven't changed since the last time we looked.
+   */
+  private final Map<String, SnapshotDirectoryInfo> snapshots =
+      new HashMap<String, SnapshotDirectoryInfo>();
+  private final Timer refreshTimer;
+
+  private long lastModifiedTime = Long.MIN_VALUE;
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem.
+   * <p>
+   * Immediately loads the file cache.
+   * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
+   *          hbase root directory
+   * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
+   */
+  public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
+      SnapshotFileInspector inspectSnapshotFiles) throws IOException {
+    this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
+        refreshThreadName, inspectSnapshotFiles);
+  }
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem
+   * @param fs {@link FileSystem} where the snapshots are stored
+   * @param rootDir hbase root directory
+   * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
+   * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   */
+  public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
+      long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
+    this.fs = fs;
+    this.fileInspector = inspectSnapshotFiles;
+    this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    // periodically refresh the file cache to make sure we aren't superfluously saving files.
+    this.refreshTimer = new Timer(refreshThreadName, true);
+    this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
+      cacheRefreshPeriod);
+  }
+
+  /**
+   * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
+   * cache refreshes.
+   * <p>
+   * Blocks until the cache is refreshed.
+   * <p>
+   * Exposed for TESTING.
+   */
+  public void triggerCacheRefreshForTesting() {
+    try {
+      SnapshotFileCache.this.refreshCache();
+    } catch (IOException e) {
+      LOG.warn("Failed to refresh snapshot hfile cache!", e);
+    }
+    LOG.debug("Current cache:" + cache);
+  }
+
+  /**
+   * Check to see if the passed file name is contained in any of the snapshots. First checks an
+   * in-memory cache of the files to keep. If its not in the cache, then the cache is refreshed and
+   * the cache checked again for that file. This ensures that we always return <tt>true</tt> for a
+   * files that exists.
+   * <p>
+   * Note this may lead to periodic false positives for the file being referenced. Periodically, the
+   * cache is refreshed even if there are no requests to ensure that the false negatives get removed
+   * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
+   * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
+   * at that point, cache will still think the file system contains that file and return
+   * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
+   * on the filesystem, we will never find it and always return <tt>false</tt>.
+   * @param fileName file to check
+   * @return <tt>false</tt> if the file is not referenced in any current or running snapshot,
+   *         <tt>true</tt> if the file is in the cache.
+   * @throws IOException if there is an unexpected error reaching the filesystem.
+   */
+  // XXX this is inefficient to synchronize on the method, when what we really need to guard against
+  // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
+  // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
+  public synchronized boolean contains(String fileName) throws IOException {
+    if (this.cache.contains(fileName)) return true;
+
+    refreshCache();
+
+    // then check again
+    return this.cache.contains(fileName);
+  }
+
+  private synchronized void refreshCache() throws IOException {
+    // get the status of the snapshots directory
+    FileStatus status;
+    try {
+      status = fs.getFileStatus(snapshotDir);
+    } catch (FileNotFoundException e) {
+      LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+      return;
+    }
+    // if the snapshot directory wasn't modified since we last check, we are done
+    if (status.getModificationTime() <= lastModifiedTime) return;
+
+    // directory was modified, so we need to reload our cache
+    // there could be a slight race here where we miss the cache, check the directory modification
+    // time, then someone updates the directory, causing us to not scan the directory again.
+    // However, snapshot directories are only created once, so this isn't an issue.
+
+    // 1. update the modified time
+    this.lastModifiedTime = status.getModificationTime();
+
+    // 2.clear the cache
+    this.cache.clear();
+    Map<String, SnapshotDirectoryInfo> known = new HashMap<String, SnapshotDirectoryInfo>();
+
+    // 3. check each of the snapshot directories
+    FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
+    if (snapshots == null) {
+      // remove all the remembered snapshots because we don't have any left
+      LOG.debug("No snapshots on-disk, cache empty");
+      this.snapshots.clear();
+      return;
+    }
+
+    // 3.1 iterate through the on-disk snapshots
+    for (FileStatus snapshot : snapshots) {
+      String name = snapshot.getPath().getName();
+      // its the tmp dir
+      if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+        // only add those files to the cache, but not to the known snapshots
+        FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
+        if (running == null) continue;
+        for (FileStatus run : running) {
+          this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        }
+      } else {
+        SnapshotDirectoryInfo files = this.snapshots.remove(name);
+        // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
+        // the latter could occur where I create a snapshot, then delete it, and then make a new
+        // snapshot with the same name. We will need to update the cache the information from that new
+        // snapshot, even though it has the same name as the files referenced have probably changed.
+        if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
+          // get all files for the snapshot and create a new info
+          Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
+          files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
+        }
+        // 3.2 add all the files to cache
+        this.cache.addAll(files.getFiles());
+        known.put(name, files);
+      }
+    }
+
+    // 4. set the snapshots we are tracking
+    this.snapshots.clear();
+    this.snapshots.putAll(known);
+  }
+
+  /**
+   * Simple helper task that just periodically attempts to refresh the cache
+   */
+  public class RefreshCacheTask extends TimerTask {
+    @Override
+    public void run() {
+      try {
+        SnapshotFileCache.this.refreshCache();
+      } catch (IOException e) {
+        LOG.warn("Failed to refresh snapshot hfile cache!", e);
+      }
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    if (!this.stop) {
+      this.stop = true;
+      this.refreshTimer.cancel();
+    }
+
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stop;
+  }
+
+  /**
+   * Information about a snapshot directory
+   */
+  private static class SnapshotDirectoryInfo {
+    long lastModified;
+    Collection<String> files;
+
+    public SnapshotDirectoryInfo(long mtime, Collection<String> files) {
+      this.lastModified = mtime;
+      this.files = files;
+    }
+
+    /**
+     * @return the hfiles in the snapshot when <tt>this</tt> was made.
+     */
+    public Collection<String> getFiles() {
+      return this.files;
+    }
+
+    /**
+     * Check if the snapshot directory has been modified
+     * @param mtime current modification time of the directory
+     * @return <tt>true</tt> if it the modification time of the directory is newer time when we
+     *         created <tt>this</tt>
+     */
+    public boolean hasBeenModified(long mtime) {
+      return this.lastModified < mtime;
+    }
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
+ * tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hfilecleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
+
+  /** File cache for HFiles in the completed and currently running snapshots */
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    try {
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
+        DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create cleaner util", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+
+  /**
+   * Exposed for Testing!
+   * @return the cache of all hfiles
+   */
+  public SnapshotFileCache getFileCacheForTesting() {
+    return this.cache;
+  }
+}

Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java Mon Mar  4 11:24:50 2013
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a log cleaner that checks if a log is still used by
+ * snapshots of HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotLogCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  static final String HLOG_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hlogcleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HLOG_CACHE_REFRESH_PERIOD = 300000;
+
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isLogDeletable(Path filePath) {
+    try {
+      if (null == cache) return false;
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  /**
+   * This method should only be called <b>once</b>, as it starts a thread to keep the cache
+   * up-to-date.
+   * <p>
+   * {@inheritDoc}
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(
+        HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create snapshot log cleaner", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+}



Mime
View raw message