hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [41/50] [abbrv] hbase git commit: HBASE-14439 Move fs stuff out pt2 - region fs
Date Thu, 22 Sep 2016 16:34:29 GMT
HBASE-14439 Move fs stuff out pt2 - region fs


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fe2918ba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fe2918ba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fe2918ba

Branch: refs/heads/hbase-14439
Commit: fe2918ba148701ba7b166fd1da9b6a85cc645a8b
Parents: 6796171
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Fri Oct 9 11:47:27 2015 -0700
Committer: Sean Busbey <busbey@apache.org>
Committed: Thu Sep 22 11:19:52 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HTableDescriptor.java   |  54 +-
 .../hadoop/hbase/backup/HFileArchiver.java      | 690 -----------------
 .../hadoop/hbase/fs/FSUtilsWithRetries.java     | 178 +++++
 .../hadoop/hbase/fs/MasterFileSystem.java       | 102 ++-
 .../hadoop/hbase/fs/RegionFileSystem.java       | 183 +++++
 .../hbase/fs/legacy/HFileArchiveUtil.java       | 156 ++++
 .../hadoop/hbase/fs/legacy/HFileArchiver.java   | 690 +++++++++++++++++
 .../hadoop/hbase/fs/legacy/LegacyArchiver.java  |  29 +
 .../hadoop/hbase/fs/legacy/LegacyLayout.java    |  52 +-
 .../hbase/fs/legacy/LegacyMasterFileSystem.java |  29 +-
 .../hbase/fs/legacy/LegacyRegionFileSystem.java | 758 +++++++++++++++++++
 .../hadoop/hbase/master/RegionStates.java       |   5 +-
 .../hbase/regionserver/StoreFileInfo.java       | 104 ++-
 .../hadoop/hbase/util/FSRegionScanner.java      | 104 +--
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 388 ++--------
 .../org/apache/hadoop/hbase/util/FSVisitor.java | 109 ---
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  89 ++-
 .../hadoop/hbase/util/HFileArchiveUtil.java     | 156 ----
 .../TestCorruptedRegionStoreFile.java           |  15 +-
 .../TestSplitTransactionOnCluster.java          |  27 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |   1 -
 .../apache/hadoop/hbase/util/TestFSVisitor.java | 123 ---
 22 files changed, 2384 insertions(+), 1658 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 867dc0d..8897e35 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1437,23 +1437,47 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
       remove(match);
   }
 
-  /**
-   * Returns the {@link Path} object representing the table directory under
-   * path rootdir
-   *
-   * Deprecated use FSUtils.getTableDir() instead.
-   *
-   * @param rootdir qualified path of HBase root directory
-   * @param tableName name of table
-   * @return {@link Path} for table
+  /** Table descriptor for <code>hbase:meta</code> catalog table
+   * Deprecated, use TableDescriptors#get(TableName.META_TABLE) or
+   * Admin#getTableDescriptor(TableName.META_TABLE) instead.
    */
   @Deprecated
-  public static Path getTableDir(Path rootdir, final byte [] tableName) {
-    //This is bad I had to mirror code from FSUTils.getTableDir since
-    //there is no module dependency between hbase-client and hbase-server
-    TableName name = TableName.valueOf(tableName);
-    return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
-              new Path(name.getNamespaceAsString(), new Path(name.getQualifierAsString()))));
+  public static final HTableDescriptor META_TABLEDESC = new HTableDescriptor(
+      TableName.META_TABLE_NAME,
+      new HColumnDescriptor[] {
+          new HColumnDescriptor(HConstants.CATALOG_FAMILY)
+              // Ten is arbitrary number.  Keep versions to help debugging.
+              .setMaxVersions(10)
+              .setInMemory(true)
+              .setBlocksize(8 * 1024)
+              .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+              // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+              .setBloomFilterType(BloomType.NONE)
+              // Enable cache of data blocks in L1 if more than one caching tier deployed:
+              // e.g. if using CombinedBlockCache (BucketCache).
+              .setCacheDataInL1(true),
+          new HColumnDescriptor(HConstants.TABLE_FAMILY)
+              // Ten is arbitrary number.  Keep versions to help debugging.
+              .setMaxVersions(10)
+              .setInMemory(true)
+              .setBlocksize(8 * 1024)
+              .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                  // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+              .setBloomFilterType(BloomType.NONE)
+                  // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                  // e.g. if using CombinedBlockCache (BucketCache).
+              .setCacheDataInL1(true)
+      });
+
+  static {
+    try {
+      META_TABLEDESC.addCoprocessor(
+          "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint",
+          null, Coprocessor.PRIORITY_SYSTEM, null);
+    } catch (IOException ex) {
+      //LOG.warn("exception in loading coprocessor for the hbase:meta table");
+      throw new RuntimeException(ex);
+    }
   }
 
   public final static String NAMESPACE_FAMILY_INFO = "info";

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
deleted file mode 100644
index d682ccc..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ /dev/null
@@ -1,690 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hadoop.io.MultipleIOException;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-
-/**
- * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
- * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
- * the state of the system.
- */
-public class HFileArchiver {
-  private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
-  private static final String SEPARATOR = ".";
-
-  /** Number of retries in case of fs operation failure */
-  private static final int DEFAULT_RETRIES_NUMBER = 3;
-
-  private HFileArchiver() {
-    // hidden ctor since this is just a util
-  }
-
-  /**
-   * Cleans up all the files for a HRegion by archiving the HFiles to the
-   * archive directory
-   * @param conf the configuration to use
-   * @param fs the file system object
-   * @param info HRegionInfo for region to be deleted
-   * @throws IOException
-   */
-  public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info)
-      throws IOException {
-    Path rootDir = FSUtils.getRootDir(conf);
-    archiveRegion(fs, rootDir, FSUtils.getTableDir(rootDir, info.getTable()),
-      HRegion.getRegionDir(rootDir, info));
-  }
-
-  /**
-   * Remove an entire region from the table directory via archiving the region's hfiles.
-   * @param fs {@link FileSystem} from which to remove the region
-   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
-   *          the archive path)
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param regionDir {@link Path} to where a region is being stored (for building the archive path)
-   * @return <tt>true</tt> if the region was sucessfully deleted. <tt>false</tt> if the filesystem
-   *         operations could not complete.
-   * @throws IOException if the request cannot be completed
-   */
-  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
-      throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("ARCHIVING " + regionDir.toString());
-    }
-
-    // otherwise, we archive the files
-    // make sure we can archive
-    if (tableDir == null || regionDir == null) {
-      LOG.error("No archive directory could be found because tabledir (" + tableDir
-          + ") or regiondir (" + regionDir + "was null. Deleting files instead.");
-      deleteRegionWithoutArchiving(fs, regionDir);
-      // we should have archived, but failed to. Doesn't matter if we deleted
-      // the archived files correctly or not.
-      return false;
-    }
-
-    // make sure the regiondir lives under the tabledir
-    Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
-    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir,
-        FSUtils.getTableName(tableDir),
-        regionDir.getName());
-
-    FileStatusConverter getAsFile = new FileStatusConverter(fs);
-    // otherwise, we attempt to archive the store files
-
-    // build collection of just the store directories to archive
-    Collection<File> toArchive = new ArrayList<File>();
-    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
-    PathFilter nonHidden = new PathFilter() {
-      @Override
-      public boolean accept(Path file) {
-        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
-      }
-    };
-    FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
-    // if there no files, we can just delete the directory and return;
-    if (storeDirs == null) {
-      LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!");
-      return deleteRegionWithoutArchiving(fs, regionDir);
-    }
-
-    // convert the files in the region to a File
-    toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile));
-    LOG.debug("Archiving " + toArchive);
-    boolean success = false;
-    try {
-      success = resolveAndArchive(fs, regionArchiveDir, toArchive);
-    } catch (IOException e) {
-      LOG.error("Failed to archive " + toArchive, e);
-      success = false;
-    }
-
-    // if that was successful, then we delete the region
-    if (success) {
-      return deleteRegionWithoutArchiving(fs, regionDir);
-    }
-
-    throw new IOException("Received error when attempting to archive files (" + toArchive
-        + "), cannot delete region directory. ");
-  }
-
-  /**
-   * Remove from the specified region the store files of the specified column family,
-   * either by archiving them or outright deletion
-   * @param fs the filesystem where the store files live
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param parent Parent region hosting the store files
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param family the family hosting the store files
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveFamily(FileSystem fs, Configuration conf,
-      HRegionInfo parent, Path tableDir, byte[] family) throws IOException {
-    Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family)));
-    FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir);
-    if (storeFiles == null) {
-      LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() +
-          ", family=" + Bytes.toString(family));
-      return;
-    }
-
-    FileStatusConverter getAsFile = new FileStatusConverter(fs);
-    Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile);
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family);
-
-    // do the actual archive
-    if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) {
-      throw new IOException("Failed to archive/delete all the files for region:"
-          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Remove the store files, either by archiving them or outright deletion
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param fs the filesystem where the store files live
-   * @param regionInfo {@link HRegionInfo} of the region hosting the store files
-   * @param family the family hosting the store files
-   * @param compactedFiles files to be disposed of. No further reading of these files should be
-   *          attempted; otherwise likely to cause an {@link IOException}
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
-      Path tableDir, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
-
-    // sometimes in testing, we don't have rss, so we need to check for that
-    if (fs == null) {
-      LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:"
-          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family));
-      deleteStoreFilesWithoutArchiving(compactedFiles);
-      return;
-    }
-
-    // short circuit if we don't have any files to delete
-    if (compactedFiles.size() == 0) {
-      LOG.debug("No store files to dispose, done!");
-      return;
-    }
-
-    // build the archive path
-    if (regionInfo == null || family == null) throw new IOException(
-        "Need to have a region and a family to archive from.");
-
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
-
-    // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
-      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
-          + Bytes.toString(family) + ", deleting compacted files instead.");
-    }
-
-    // otherwise we attempt to archive the store files
-    if (LOG.isDebugEnabled()) LOG.debug("Archiving compacted store files.");
-
-    // Wrap the storefile into a File
-    StoreToFile getStorePath = new StoreToFile(fs);
-    Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
-
-    // do the actual archive
-    if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
-      throw new IOException("Failed to archive/delete all the files for region:"
-          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Archive the store file
-   * @param fs the filesystem where the store files live
-   * @param regionInfo region hosting the store files
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param family the family hosting the store files
-   * @param storeFile file to be archived
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
-      Path tableDir, byte[] family, Path storeFile) throws IOException {
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
-    // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
-      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
-          + Bytes.toString(family) + ", deleting compacted files instead.");
-    }
-
-    // do the actual archive
-    long start = EnvironmentEdgeManager.currentTime();
-    File file = new FileablePath(fs, storeFile);
-    if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
-      throw new IOException("Failed to archive/delete the file for region:"
-          + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Archive the given files and resolve any conflicts with existing files via appending the time
-   * archiving started (so all conflicts in the same group have the same timestamp appended).
-   * <p>
-   * If any of the passed files to archive are directories, archives all the files under that
-   * directory. Archive directory structure for children is the base archive directory name + the
-   * parent directory and is built recursively is passed files are directories themselves.
-   * @param fs {@link FileSystem} on which to archive the files
-   * @param baseArchiveDir base archive directory to archive the given files
-   * @param toArchive files to be archived
-   * @return <tt>true</tt> on success, <tt>false</tt> otherwise
-   * @throws IOException on unexpected failure
-   */
-  private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
-      Collection<File> toArchive) throws IOException {
-    if (LOG.isTraceEnabled()) LOG.trace("Starting to archive " + toArchive);
-    long start = EnvironmentEdgeManager.currentTime();
-    List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
-
-    // notify that some files were not archived.
-    // We can't delete the files otherwise snapshots or other backup system
-    // that relies on the archiver end up with data loss.
-    if (failures.size() > 0) {
-      LOG.warn("Failed to complete archive of: " + failures +
-        ". Those files are still in the original location, and they may slow down reads.");
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Resolve any conflict with an existing archive file via timestamp-append
-   * renaming of the existing file and then archive the passed in files.
-   * @param fs {@link FileSystem} on which to archive the files
-   * @param baseArchiveDir base archive directory to store the files. If any of
-   *          the files to archive are directories, will append the name of the
-   *          directory to the base archive directory name, creating a parallel
-   *          structure.
-   * @param toArchive files/directories that need to be archvied
-   * @param start time the archiving started - used for resolving archive
-   *          conflicts.
-   * @return the list of failed to archive files.
-   * @throws IOException if an unexpected file operation exception occured
-   */
-  private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
-      Collection<File> toArchive, long start) throws IOException {
-    // short circuit if no files to move
-    if (toArchive.size() == 0) return Collections.emptyList();
-
-    if (LOG.isTraceEnabled()) LOG.trace("moving files to the archive directory: " + baseArchiveDir);
-
-    // make sure the archive directory exists
-    if (!fs.exists(baseArchiveDir)) {
-      if (!fs.mkdirs(baseArchiveDir)) {
-        throw new IOException("Failed to create the archive directory:" + baseArchiveDir
-            + ", quitting archive attempt.");
-      }
-      if (LOG.isTraceEnabled()) LOG.trace("Created archive directory:" + baseArchiveDir);
-    }
-
-    List<File> failures = new ArrayList<File>();
-    String startTime = Long.toString(start);
-    for (File file : toArchive) {
-      // if its a file archive it
-      try {
-        if (LOG.isTraceEnabled()) LOG.trace("Archiving: " + file);
-        if (file.isFile()) {
-          // attempt to archive the file
-          if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) {
-            LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir);
-            failures.add(file);
-          }
-        } else {
-          // otherwise its a directory and we need to archive all files
-          if (LOG.isTraceEnabled()) LOG.trace(file + " is a directory, archiving children files");
-          // so we add the directory name to the one base archive
-          Path parentArchiveDir = new Path(baseArchiveDir, file.getName());
-          // and then get all the files from that directory and attempt to
-          // archive those too
-          Collection<File> children = file.getChildren();
-          failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start));
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed to archive " + file, e);
-        failures.add(file);
-      }
-    }
-    return failures;
-  }
-
-  /**
-   * Attempt to archive the passed in file to the archive directory.
-   * <p>
-   * If the same file already exists in the archive, it is moved to a timestamped directory under
-   * the archive directory and the new file is put in its place.
-   * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles
-   * @param currentFile {@link Path} to the original HFile that will be archived
-   * @param archiveStartTime time the archiving started, to resolve naming conflicts
-   * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a
-   *         problem, but the operation still completed.
-   * @throws IOException on failure to complete {@link FileSystem} operations.
-   */
-  private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
-      String archiveStartTime) throws IOException {
-    // build path as it should be in the archive
-    String filename = currentFile.getName();
-    Path archiveFile = new Path(archiveDir, filename);
-    FileSystem fs = currentFile.getFileSystem();
-
-    // if the file already exists in the archive, move that one to a timestamped backup. This is a
-    // really, really unlikely situtation, where we get the same name for the existing file, but
-    // is included just for that 1 in trillion chance.
-    if (fs.exists(archiveFile)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("File:" + archiveFile + " already exists in archive, moving to "
-            + "timestamped backup and overwriting current.");
-      }
-
-      // move the archive file to the stamped backup
-      Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!fs.rename(archiveFile, backedupArchiveFile)) {
-        LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
-            + ", deleting existing file in favor of newer.");
-        // try to delete the exisiting file, if we can't rename it
-        if (!fs.delete(archiveFile, false)) {
-          throw new IOException("Couldn't delete existing archive file (" + archiveFile
-              + ") or rename it to the backup file (" + backedupArchiveFile
-              + ") to make room for similarly named file.");
-        }
-      }
-      LOG.debug("Backed up archive file from " + archiveFile);
-    }
-
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("No existing file in archive for: " + archiveFile +
-        ", free to archive original file.");
-    }
-
-    // at this point, we should have a free spot for the archive file
-    boolean success = false;
-    for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) {
-      if (i > 0) {
-        // Ensure that the archive directory exists.
-        // The previous "move to archive" operation has failed probably because
-        // the cleaner has removed our archive directory (HBASE-7643).
-        // (we're in a retry loop, so don't worry too much about the exception)
-        try {
-          if (!fs.exists(archiveDir)) {
-            if (fs.mkdirs(archiveDir)) {
-              LOG.debug("Created archive directory:" + archiveDir);
-            }
-          }
-        } catch (IOException e) {
-          LOG.warn("Failed to create directory: " + archiveDir, e);
-        }
-      }
-
-      try {
-        success = currentFile.moveAndClose(archiveFile);
-      } catch (IOException e) {
-        LOG.warn("Failed to archive " + currentFile + " on try #" + i, e);
-        success = false;
-      }
-    }
-
-    if (!success) {
-      LOG.error("Failed to archive " + currentFile);
-      return false;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Finished archiving from " + currentFile + ", to " + archiveFile);
-    }
-    return true;
-  }
-
-  /**
-   * Without regard for backup, delete a region. Should be used with caution.
-   * @param regionDir {@link Path} to the region to be deleted.
-   * @param fs FileSystem from which to delete the region
-   * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise
-   * @throws IOException on filesystem operation failure
-   */
-  private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
-      throws IOException {
-    if (fs.delete(regionDir, true)) {
-      LOG.debug("Deleted all region files in: " + regionDir);
-      return true;
-    }
-    LOG.debug("Failed to delete region directory:" + regionDir);
-    return false;
-  }
-
-  /**
-   * Just do a simple delete of the given store files
-   * <p>
-   * A best effort is made to delete each of the files, rather than bailing on the first failure.
-   * <p>
-   * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
-   * less resources, but is limited in terms of usefulness
-   * @param compactedFiles store files to delete from the file system.
-   * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
-   *           throwing the exception, rather than failing at the first file.
-   */
-  private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
-      throws IOException {
-    LOG.debug("Deleting store files without archiving.");
-    List<IOException> errors = new ArrayList<IOException>(0);
-    for (StoreFile hsf : compactedFiles) {
-      try {
-        hsf.deleteReader();
-      } catch (IOException e) {
-        LOG.error("Failed to delete store file:" + hsf.getPath());
-        errors.add(e);
-      }
-    }
-    if (errors.size() > 0) {
-      throw MultipleIOException.createIOException(errors);
-    }
-  }
-
-  /**
-   * Adapt a type to match the {@link File} interface, which is used internally for handling
-   * archival/removal of files
-   * @param <T> type to adapt to the {@link File} interface
-   */
-  private static abstract class FileConverter<T> implements Function<T, File> {
-    protected final FileSystem fs;
-
-    public FileConverter(FileSystem fs) {
-      this.fs = fs;
-    }
-  }
-
-  /**
-   * Convert a FileStatus to something we can manage in the archiving
-   */
-  private static class FileStatusConverter extends FileConverter<FileStatus> {
-    public FileStatusConverter(FileSystem fs) {
-      super(fs);
-    }
-
-    @Override
-    public File apply(FileStatus input) {
-      return new FileablePath(fs, input.getPath());
-    }
-  }
-
-  /**
-   * Convert the {@link StoreFile} into something we can manage in the archive
-   * methods
-   */
-  private static class StoreToFile extends FileConverter<StoreFile> {
-    public StoreToFile(FileSystem fs) {
-      super(fs);
-    }
-
-    @Override
-    public File apply(StoreFile input) {
-      return new FileableStoreFile(fs, input);
-    }
-  }
-
-  /**
-   * Wrapper to handle file operations uniformly
-   */
-  private static abstract class File {
-    protected final FileSystem fs;
-
-    public File(FileSystem fs) {
-      this.fs = fs;
-    }
-
-    /**
-     * Delete the file
-     * @throws IOException on failure
-     */
-    abstract void delete() throws IOException;
-
-    /**
-     * Check to see if this is a file or a directory
-     * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise
-     * @throws IOException on {@link FileSystem} connection error
-     */
-    abstract boolean isFile() throws IOException;
-
-    /**
-     * @return if this is a directory, returns all the children in the
-     *         directory, otherwise returns an empty list
-     * @throws IOException
-     */
-    abstract Collection<File> getChildren() throws IOException;
-
-    /**
-     * close any outside readers of the file
-     * @throws IOException
-     */
-    abstract void close() throws IOException;
-
-    /**
-     * @return the name of the file (not the full fs path, just the individual
-     *         file name)
-     */
-    abstract String getName();
-
-    /**
-     * @return the path to this file
-     */
-    abstract Path getPath();
-
-    /**
-     * Move the file to the given destination
-     * @param dest
-     * @return <tt>true</tt> on success
-     * @throws IOException
-     */
-    public boolean moveAndClose(Path dest) throws IOException {
-      this.close();
-      Path p = this.getPath();
-      return FSUtils.renameAndSetModifyTime(fs, p, dest);
-    }
-
-    /**
-     * @return the {@link FileSystem} on which this file resides
-     */
-    public FileSystem getFileSystem() {
-      return this.fs;
-    }
-
-    @Override
-    public String toString() {
-      return this.getClass() + ", file:" + getPath().toString();
-    }
-  }
-
-  /**
-   * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}.
-   */
-  private static class FileablePath extends File {
-    private final Path file;
-    private final FileStatusConverter getAsFile;
-
-    public FileablePath(FileSystem fs, Path file) {
-      super(fs);
-      this.file = file;
-      this.getAsFile = new FileStatusConverter(fs);
-    }
-
-    @Override
-    public void delete() throws IOException {
-      if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
-    }
-
-    @Override
-    public String getName() {
-      return file.getName();
-    }
-
-    @Override
-    public Collection<File> getChildren() throws IOException {
-      if (fs.isFile(file)) return Collections.emptyList();
-      return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile);
-    }
-
-    @Override
-    public boolean isFile() throws IOException {
-      return fs.isFile(file);
-    }
-
-    @Override
-    public void close() throws IOException {
-      // NOOP - files are implicitly closed on removal
-    }
-
-    @Override
-    Path getPath() {
-      return file;
-    }
-  }
-
-  /**
-   * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
-   * .
-   */
-  private static class FileableStoreFile extends File {
-    StoreFile file;
-
-    public FileableStoreFile(FileSystem fs, StoreFile store) {
-      super(fs);
-      this.file = store;
-    }
-
-    @Override
-    public void delete() throws IOException {
-      file.deleteReader();
-    }
-
-    @Override
-    public String getName() {
-      return file.getPath().getName();
-    }
-
-    @Override
-    public boolean isFile() {
-      return true;
-    }
-
-    @Override
-    public Collection<File> getChildren() throws IOException {
-      // storefiles don't have children
-      return Collections.emptyList();
-    }
-
-    @Override
-    public void close() throws IOException {
-      file.closeReader(true);
-    }
-
-    @Override
-    Path getPath() {
-      return file.getPath();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
new file mode 100644
index 0000000..5dad2fb
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
@@ -0,0 +1,178 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class FSUtilsWithRetries {
+  private static final Log LOG = LogFactory.getLog(FSUtilsWithRetries.class);
+
+  /**
+   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
+   * client level.
+   */
+  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
+  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
+  private final int hdfsClientRetriesNumber;
+  private final int baseSleepBeforeRetries;
+
+  private final FileSystem fs;
+
+  public FSUtilsWithRetries(final Configuration conf, final FileSystem fs) {
+    this.fs = fs;
+
+    this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
+      DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
+    this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
+      DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
+  }
+
+  public FileSystem getFileSystem() {
+    return fs;
+  }
+
+  public boolean exists(Path path) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.exists(path);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(path)) return true; // directory is present
+        try {
+          sleepBeforeRetry("Check existence", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in check existence", lastIOE);
+  }
+
+  /**
+   * Creates a directory. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
+   *         whether the directory exists or not, and returns true if it exists.
+   * @throws IOException
+   */
+  public boolean createDir(Path dir) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.mkdirs(dir);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(dir)) return true; // directory is present
+        try {
+          sleepBeforeRetry("Create Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createDir", lastIOE);
+  }
+
+  /**
+   * Renames a directory. Assumes the user has already checked for this directory existence.
+   * @param srcpath
+   * @param dstPath
+   * @return true if rename is successful.
+   * @throws IOException
+   */
+  public boolean rename(Path srcpath, Path dstPath) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.rename(srcpath, dstPath);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
+        // dir is not there, retry after some time.
+        try {
+          sleepBeforeRetry("Rename Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+
+    throw new IOException("Exception in rename", lastIOE);
+  }
+
+  /**
+   * Deletes a directory. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @return true if the directory is deleted.
+   * @throws IOException
+   */
+  public boolean deleteDir(Path dir) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.delete(dir, true);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(dir)) return true;
+        // dir is there, retry deleting after some time.
+        try {
+          sleepBeforeRetry("Delete Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+
+    throw new IOException("Exception in DeleteDir", lastIOE);
+  }
+
+  /**
+   * sleeping logic; handles the interrupt exception.
+   */
+  private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
+    sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
+  }
+
+  /**
+   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
+   * for this to avoid re-looking for the integer values.
+   */
+  private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
+      int hdfsClientRetriesNumber) throws InterruptedException {
+    if (sleepMultiplier > hdfsClientRetriesNumber) {
+      LOG.debug(msg + ", retries exhausted");
+      return;
+    }
+    LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
+    Thread.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
index 2a58b9e..020917a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -142,6 +143,69 @@ public abstract class MasterFileSystem {
   }
 
   // ==========================================================================
+  //  PUBLIC Methods - Table Region related
+  // ==========================================================================
+  public void deleteRegion(HRegionInfo regionInfo) throws IOException {
+    RegionFileSystem.destroy(conf, fs, rootDir, regionInfo);
+  }
+
+  public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+    return getRegions(FsContext.DATA, tableName);
+  }
+
+  public abstract Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
+    throws IOException;
+
+  // TODO: Move in HRegionFileSystem
+  public void deleteFamilyFromFS(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
+      throws IOException {
+    getRegionFileSystem(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public RegionFileSystem getRegionFileSystem(HRegionInfo regionInfo) throws IOException {
+    return RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - visitors
+  // ==========================================================================
+  public void visitStoreFiles(StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, visitor);
+  }
+
+  public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, namespace, visitor);
+  }
+
+  public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, table, visitor);
+  }
+
+  public void visitStoreFiles(FsContext ctx, StoreFileVisitor visitor)
+      throws IOException {
+    for (String namespace: getNamespaces()) {
+      visitStoreFiles(ctx, namespace, visitor);
+    }
+  }
+
+  public void visitStoreFiles(FsContext ctx, String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    for (TableName tableName: getTables(namespace)) {
+      visitStoreFiles(ctx, tableName, visitor);
+    }
+  }
+
+  public void visitStoreFiles(FsContext ctx, TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    for (HRegionInfo hri: getRegions(ctx, table)) {
+      RegionFileSystem.open(conf, fs, rootDir, hri, false).visitStoreFiles(visitor);
+    }
+  }
+
+  // ==========================================================================
   //  PUBLIC Methods - bootstrap
   // ==========================================================================
   public abstract Path getTempDir();
@@ -244,44 +308,6 @@ public abstract class MasterFileSystem {
     bootstrapMeta();
   }
 
-  // TODO: Move in HRegionFileSystem
-  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
-      throws IOException {
-    // archive family store files
-    Path tableDir = FSUtils.getTableDir(getRootDir(), region.getTable());
-    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
-
-    // delete the family folder
-    Path familyDir = new Path(tableDir,
-      new Path(region.getEncodedName(), Bytes.toString(familyName)));
-    if (fs.delete(familyDir, true) == false) {
-      if (fs.exists(familyDir)) {
-        throw new IOException("Could not delete family "
-            + Bytes.toString(familyName) + " from FileSystem for region "
-            + region.getRegionNameAsString() + "(" + region.getEncodedName()
-            + ")");
-      }
-    }
-
-    // archive and delete mob files
-    if (hasMob) {
-      Path mobTableDir =
-          FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
-      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
-      Path mobFamilyDir =
-          new Path(mobTableDir,
-              new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
-      // archive mob family store files
-      MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
-
-      if (!fs.delete(mobFamilyDir, true)) {
-        throw new IOException("Could not delete mob store files for family "
-            + Bytes.toString(familyName) + " from FileSystem region "
-            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
-      }
-    }
-  }
-
   // ==========================================================================
   //  PUBLIC
   // ==========================================================================

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
new file mode 100644
index 0000000..3b94355
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
@@ -0,0 +1,183 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+@InterfaceAudience.Private
+public abstract class RegionFileSystem {
+  private static Log LOG = LogFactory.getLog(RegionFileSystem.class);
+
+  private final Configuration conf;
+  private final HRegionInfo hri;
+  private final FileSystem fs;
+  private final Path rootDir;
+
+  protected RegionFileSystem(Configuration conf, FileSystem fs, Path rootDir, HRegionInfo hri) {
+    this.conf = conf;
+    this.rootDir = rootDir;
+    this.hri = hri;
+    this.fs = fs;
+  }
+
+  public Configuration getConfiguration() { return conf; }
+  public FileSystem getFileSystem() { return fs; }
+  public Path getRootDir() { return rootDir; }
+
+  public HRegionInfo getRegionInfo() { return hri; }
+  public TableName getTable() { return getRegionInfo().getTable(); }
+
+  // ==========================================================================
+  //  PUBLIC Interfaces - Visitors
+  // ==========================================================================
+  public interface StoreFileVisitor {
+    void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+       throws IOException;
+  }
+
+  public void visitStoreFiles(final StoreFileVisitor visitor) throws IOException {
+    for (String familyName: getFamilies()) {
+      for (StoreFileInfo storeFile: getStoreFiles(familyName)) {
+        visitor.storeFile(getRegionInfo(), familyName, storeFile);
+      }
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Families Related
+  // ==========================================================================
+
+  /**
+   * @return the set of families present on disk
+   * @throws IOException
+   */
+  public abstract Collection<String> getFamilies() throws IOException;
+
+  public void deleteFamily(byte[] familyName, boolean hasMob) throws IOException {
+    deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public abstract void deleteFamily(String familyName, boolean hasMob) throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC Methods - Store Files related
+  // ==========================================================================
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
+    return getStoreFiles(Bytes.toString(familyName));
+  }
+
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+    return getStoreFiles(familyName, true);
+  }
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate)
+      throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC bootstrap
+  // ==========================================================================
+  protected abstract void bootstrap() throws IOException;
+  protected abstract void destroy() throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC
+  // ==========================================================================
+  public static RegionFileSystem open(Configuration conf, HRegionInfo regionInfo, boolean bootstrap)
+      throws IOException {
+    return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf),
+        regionInfo, bootstrap);
+  }
+
+  public static RegionFileSystem open(Configuration conf, FileSystem fs, Path rootDir,
+      HRegionInfo regionInfo, boolean bootstrap) throws IOException {
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    fs = rootDir.getFileSystem(conf);
+    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+    // make sure the fs has the same conf
+    fs.setConf(conf);
+
+    RegionFileSystem rfs = getInstance(conf, fs, rootDir, regionInfo);
+    if (bootstrap) {
+      // TODO: are bootstrap and create two different things?
+      // should switch to bootstrap & read-only 
+      // legacy region wants to recover the .regioninfo :(
+    }
+    return rfs;
+  }
+
+  public static void destroy(Configuration conf, HRegionInfo regionInfo) throws IOException {
+    destroy(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), regionInfo);
+  }
+
+  public static void destroy(Configuration conf, FileSystem fs,
+      Path rootDir, HRegionInfo regionInfo) throws IOException {
+    getInstance(conf, fs, rootDir, regionInfo).destroy();
+  }
+
+  private static RegionFileSystem getInstance(Configuration conf, FileSystem fs,
+      Path rootDir, HRegionInfo regionInfo) throws IOException {
+    String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
+    switch (fsType) {
+      case "legacy":
+        return new LegacyRegionFileSystem(conf, fs, rootDir, regionInfo);
+      default:
+        throw new IOException("Invalid filesystem type " + fsType);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe2918ba/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
new file mode 100644
index 0000000..2fbaa92
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
+
+/**
+ * Helper class for all utilities related to archival/retrieval of HFiles
+ */
+public class HFileArchiveUtil {
+  private HFileArchiveUtil() {
+    // non-external instantiation - util class
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param tableName table name under which the store currently lives
+   * @param regionName region encoded name under which the store currently lives
+   * @param familyName name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or
+   *         <tt>null</tt> if it should not be archived
+   */
+  public static Path getStoreArchivePath(final Configuration conf,
+                                         final TableName tableName,
+      final String regionName, final String familyName) throws IOException {
+    Path tableArchiveDir = getTableArchivePath(conf, tableName);
+    return HStore.getStoreHomedir(tableArchiveDir, regionName, Bytes.toBytes(familyName));
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name.
+   * @param region parent region information under which the store currently lives
+   * @param tabledir directory for the table under which the store currently lives
+   * @param family name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or <tt>null</tt> if it should
+   *         not be archived
+   */
+  public static Path getStoreArchivePath(Configuration conf,
+                                         HRegionInfo region,
+                                         Path tabledir,
+      byte[] family) throws IOException {
+    Path rootDir = FSUtils.getRootDir(conf);
+    Path tableArchiveDir = getTableArchivePath(rootDir, region.getTable());
+    return HStore.getStoreHomedir(tableArchiveDir, region, family);
+  }
+
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param tableName the table name. Cannot be null.
+   * @param regiondir the path to the region directory. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Path rootDir,
+                                         TableName tableName,
+                                         Path regiondir) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(rootDir, tableName);
+
+    // then add on the region path under the archive
+    String encodedRegionName = regiondir.getName();
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param rootDir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableName name of the table to archive. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Path rootDir,
+                                         TableName tableName, String encodedRegionName) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(rootDir, tableName);
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Get the path to the table's archive directory.
+   * <p>
+   * Generally of the form: /hbase/.archive/[tablename]
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Path rootdir, final TableName tableName) {
+    return FSUtils.getTableDir(getArchivePath(rootdir), tableName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Assumed that the table should already be archived.
+   * @param conf {@link Configuration} to read the archive directory property. Can be null
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Configuration conf,
+                                         final TableName tableName)
+      throws IOException {
+    return FSUtils.getTableDir(getArchivePath(conf), tableName);
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured 
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
+   *          testing: requires a FileSystem root directory to be specified.
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   * @throws IOException if an unexpected error occurs
+   */
+  public static Path getArchivePath(Configuration conf) throws IOException {
+    return getArchivePath(FSUtils.getRootDir(conf));
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured 
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   */
+  private static Path getArchivePath(final Path rootdir) {
+    return new Path(rootdir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+}


Mime
View raw message