hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [1/3] hbase git commit: Incomplete experiment (it does not compile and has missing code)
Date Wed, 16 Sep 2015 23:53:53 GMT
Repository: hbase
Updated Branches:
  refs/heads/hbase-14439 f2856e2c5 -> e7743d779


http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
index 630ca78..be820a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
@@ -25,12 +25,14 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
 
 /**
  * Thread that walks over the filesystem, and computes the mappings
@@ -41,12 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
 class FSRegionScanner implements Runnable {
   static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
 
-  private Path regionPath;
-
-  /**
-   * The file system used
-   */
-  private FileSystem fs;
+  private HRegionFileSystem rfs;
 
   /**
    * Maps each region to the RS with highest locality for that region.
@@ -59,11 +56,10 @@ class FSRegionScanner implements Runnable {
    */
   private Map<String, Map<String, Float>> regionDegreeLocalityMapping;
 
-  FSRegionScanner(FileSystem fs, Path regionPath,
+  FSRegionScanner(Configuration conf, HRegionInfo hri,
                   Map<String, String> regionToBestLocalityRSMapping,
                   Map<String, Map<String, Float>> regionDegreeLocalityMapping) {
-    this.fs = fs;
-    this.regionPath = regionPath;
+    this.rfs = HRegionFileSystem.open(conf, hri);
     this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping;
     this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
   }
@@ -75,33 +71,13 @@ class FSRegionScanner implements Runnable {
       Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
 
       //get table name
-      String tableName = regionPath.getParent().getName();
+      String tableName = rfs.getTable();
       int totalBlkCount = 0;
 
-      // ignore null
-      FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs));
-      if (null == cfList) {
-        return;
-      }
-
-      // for each cf, get all the blocks information
-      for (FileStatus cfStatus : cfList) {
-        if (!cfStatus.isDirectory()) {
-          // skip because this is not a CF directory
-          continue;
-        }
-
-        FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
-        if (null == storeFileLists) {
-          continue;
-        }
-
-        for (FileStatus storeFile : storeFileLists) {
-          BlockLocation[] blkLocations =
-            fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
-          if (null == blkLocations) {
-            continue;
-          }
+      for (String family: rfs.getFamilies()) {
+        for (StoreFileInfo storeFile: rfs.getStoreFiles(family)) {
+          BlockLocation[] blkLocations = storeFile.getFileBlockLocations(fs);
+          if (blkLocations == null) continue;
 
           totalBlkCount += blkLocations.length;
           for(BlockLocation blk: blkLocations) {
@@ -138,7 +114,7 @@ class FSRegionScanner implements Runnable {
         if (hostToRun.endsWith(".")) {
           hostToRun = hostToRun.substring(0, hostToRun.length()-1);
         }
-        String name = tableName + ":" + regionPath.getName();
+        String name = tableName + ":" + rfs.getRegionInfo().getEncodedName();
         synchronized (regionToBestLocalityRSMapping) {
           regionToBestLocalityRSMapping.put(name,  hostToRun);
         }
@@ -157,7 +133,7 @@ class FSRegionScanner implements Runnable {
         }
         // Put the locality map into the result map, keyed by the encoded name
         // of the region.
-        regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
+        regionDegreeLocalityMapping.put(rfs.getRegionInfo().getEncodedName(), hostLocalityMap);
       }
     } catch (IOException e) {
       LOG.warn("Problem scanning file system", e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index cce37d7..426b73e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -221,6 +221,7 @@ public class FSTableDescriptors implements TableDescriptors {
     } else {
       LOG.debug("Fetching table descriptors from the filesystem.");
       boolean allvisited = true;
+      HMasterFileSystem mfs = HMasterFileSystem.open(conf, rootdir);
       for (Path d : FSUtils.getTableDirs(fs, rootdir)) {
         TableDescriptor htd = null;
         try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index ce51e27..efa8752 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -31,6 +31,7 @@ import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -64,7 +65,8 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.fs.layout.FsLayout;
+import org.apache.hadoop.hbase.fs.HMasterFileSystem;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -1027,43 +1029,6 @@ public abstract class FSUtils {
   }
 
 
-
-  /**
-   * Runs through the hbase rootdir and checks all stores have only
-   * one file in them -- that is, they've been major compacted.  Looks
-   * at root and meta tables too.
-   * @param fs filesystem
-   * @param hbaseRootDir hbase root directory
-   * @return True if this hbase install is major compacted.
-   * @throws IOException e
-   */
-  public static boolean isMajorCompacted(final FileSystem fs,
-      final Path hbaseRootDir)
-  throws IOException {
-    List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
-    RegionDirFilter regionFilter = new RegionDirFilter(fs);
-    PathFilter familyFilter = new FamilyDirFilter(fs);
-    for (Path tableDir : tableDirs) {
-      List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir, regionFilter);
-      for (FileStatus regionDir : regionDirs) {
-        Path dd = regionDir.getPath();
-        // Else its a region name.  Now look in region for families.
-        FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
-        for (FileStatus familyDir : familyDirs) {
-          Path family = familyDir.getPath();
-          // Now in family make sure only one file.
-          FileStatus[] familyStatus = fs.listStatus(family);
-          if (familyStatus.length > 1) {
-            LOG.debug(family.toString() + " has " + familyStatus.length +
-                " files.");
-            return false;
-          }
-        }
-      }
-    }
-    return true;
-  }
-
   // TODO move this method OUT of FSUtils. No dependencies to HMaster
   /**
    * Returns the total overall fragmentation percentage. Includes hbase:meta and
@@ -1109,38 +1074,32 @@ public abstract class FSUtils {
    * @throws IOException When scanning the directory fails.
    */
   public static Map<String, Integer> getTableFragmentation(
-    final FileSystem fs, final Path hbaseRootDir)
-  throws IOException {
-    Map<String, Integer> frags = new HashMap<String, Integer>();
-    int cfCountTotal = 0;
+      final FileSystem fs, final Path hbaseRootDir) throws IOException {
+    final Map<String, Integer> frags = new HashMap<String, Integer>();
+
     int cfFragTotal = 0;
-    RegionDirFilter regionFilter = new RegionDirFilter(fs);
-    PathFilter familyFilter = new FamilyDirFilter(fs);
-    List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
-    for (Path tableDir : tableDirs) {
+    int cfCountTotal = 0;
+    HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir);
+    for (TableName tableName: mfs.getTables()) {
       int cfCount = 0;
       int cfFrag = 0;
-      List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir, regionFilter);
-      for (FileStatus regionDir : regionDirs) {
-        Path dd = regionDir.getPath();
-        // else its a region name, now look in region for families
-        FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
-        for (FileStatus familyDir : familyDirs) {
+      for (HRegionInfo hri: mfs.getRegions(tableName)) {
+        HRegionFileSystem rfs = HRegionFileSystem.open(conf, hri);
+        for (String family: rfs.getFamilies()) {
           cfCount++;
           cfCountTotal++;
-          Path family = familyDir.getPath();
-          // now in family make sure only one file
-          FileStatus[] familyStatus = fs.listStatus(family);
-          if (familyStatus.length > 1) {
+          Collection<StoreFileInfo> storeFiles = rfs.getStoreFiles(family);
+          if (storeFiles.size() > 1) {
             cfFrag++;
             cfFragTotal++;
           }
         }
       }
       // compute percentage per table and store in result list
-      frags.put(FSUtils.getTableName(tableDir).getNameAsString(),
+      frags.put(tableName.getNameAsString(),
         cfCount == 0? 0: Math.round((float) cfFrag / cfCount * 100));
     }
+
     // set overall percentage for all tables
     frags.put("-TOTAL-",
       cfCountTotal == 0? 0: Math.round((float) cfFragTotal / cfCountTotal * 100));
@@ -1259,29 +1218,6 @@ public abstract class FSUtils {
   }
 
   /**
-   * A {@link PathFilter} that returns usertable directories. To get all directories use the
-   * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
-   */
-  public static class UserTableDirFilter extends BlackListDirFilter {
-    public UserTableDirFilter(FileSystem fs) {
-      super(fs, HConstants.HBASE_NON_TABLE_DIRS);
-    }
-
-    protected boolean isValidName(final String name) {
-      if (!super.isValidName(name))
-        return false;
-
-      try {
-        TableName.isLegalTableQualifierName(Bytes.toBytes(name));
-      } catch (IllegalArgumentException e) {
-        LOG.info("INVALID NAME " + name);
-        return false;
-      }
-      return true;
-    }
-  }
-
-  /**
    * Heuristic to determine whether is safe or not to open a file for append
    * Looks both for dfs.support.append and use reflection to search
    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
@@ -1336,36 +1272,6 @@ public abstract class FSUtils {
   public abstract void recoverFileLease(final FileSystem fs, final Path p,
       Configuration conf, CancelableProgressable reporter) throws IOException;
 
-  public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
-      throws IOException {
-    List<Path> tableDirs = new LinkedList<Path>();
-
-    for(FileStatus status :
-        fs.globStatus(new Path(rootdir,
-            new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) {
-      tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
-    }
-    return tableDirs;
-  }
-
-  /**
-   * @param fs
-   * @param rootdir
-   * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as
-   * .logs, .oldlogs, .corrupt folders.
-   * @throws IOException
-   */
-  public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
-      throws IOException {
-    // presumes any directory under hbase.rootdir is a table
-    FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
-    List<Path> tabledirs = new ArrayList<Path>(dirs.length);
-    for (FileStatus dir: dirs) {
-      tabledirs.add(dir.getPath());
-    }
-    return tabledirs;
-  }
-
   /**
    * Checks if the given path is the one with 'recovered.edits' dir.
    * @param path
@@ -1376,94 +1282,6 @@ public abstract class FSUtils {
   }
 
   /**
-   * Filter for all dirs that don't start with '.'
-   */
-  public static class RegionDirFilter implements PathFilter {
-    // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
-    final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
-    final FileSystem fs;
-
-    public RegionDirFilter(FileSystem fs) {
-      this.fs = fs;
-    }
-
-    @Override
-    public boolean accept(Path rd) {
-      if (!regionDirPattern.matcher(rd.getName()).matches()) {
-        return false;
-      }
-
-      try {
-        return fs.getFileStatus(rd).isDirectory();
-      } catch (IOException ioe) {
-        // Maybe the file was moved or the fs was disconnected.
-        LOG.warn("Skipping file " + rd +" due to IOException", ioe);
-        return false;
-      }
-    }
-  }
-
-  /**
-   * Filter for all dirs that are legal column family names.  This is generally used for colfam
-   * dirs &lt;hbase.rootdir&gt;/&lt;tabledir&gt;/&lt;regiondir&gt;/&lt;colfamdir&gt;.
-   */
-  public static class FamilyDirFilter implements PathFilter {
-    final FileSystem fs;
-
-    public FamilyDirFilter(FileSystem fs) {
-      this.fs = fs;
-    }
-
-    @Override
-    public boolean accept(Path rd) {
-      try {
-        // throws IAE if invalid
-        HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
-      } catch (IllegalArgumentException iae) {
-        // path name is an invalid family name and thus is excluded.
-        return false;
-      }
-
-      try {
-        return fs.getFileStatus(rd).isDirectory();
-      } catch (IOException ioe) {
-        // Maybe the file was moved or the fs was disconnected.
-        LOG.warn("Skipping file " + rd +" due to IOException", ioe);
-        return false;
-      }
-    }
-  }
-
-  /**
-   * Given a particular region dir, return all the familydirs inside it
-   *
-   * @param fs A file system for the Path
-   * @param regionDir Path to a specific region directory
-   * @return List of paths to valid family directories in region dir.
-   * @throws IOException
-   */
-  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
-    // assumes we are in a region dir.
-    FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
-    List<Path> familyDirs = new ArrayList<Path>(fds.length);
-    for (FileStatus fdfs: fds) {
-      Path fdPath = fdfs.getPath();
-      familyDirs.add(fdPath);
-    }
-    return familyDirs;
-  }
-
-  public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir) throws IOException {
-    FileStatus[] fds = fs.listStatus(familyDir, new ReferenceFileFilter(fs));
-    List<Path> referenceFiles = new ArrayList<Path>(fds.length);
-    for (FileStatus fdfs: fds) {
-      Path fdPath = fdfs.getPath();
-      referenceFiles.add(fdPath);
-    }
-    return referenceFiles;
-  }
-
-  /**
    * Filter for HFiles that excludes reference files.
    */
   public static class HFileFilter implements PathFilter {
@@ -1514,7 +1332,7 @@ public abstract class FSUtils {
    * @throws IOException
    */
   public static FileSystem getCurrentFileSystem(Configuration conf)
-  throws IOException {
+      throws IOException {
     return getRootDir(conf).getFileSystem(conf);
   }
 
@@ -1557,59 +1375,36 @@ public abstract class FSUtils {
    * @throws IOException When scanning the directory fails.
    */
   public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
-  final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors)
-  throws IOException {
+      final FileSystem fs, final Path hbaseRootDir, final TableName tableName,
+      final ErrorReporter errors) throws IOException {
+    HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir);
+    return getTableStoreFilePathMap(map, mfs, tableName, errors);
+  }
+
+  private static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
+      final HMasterFileSystem mfs, final TableName tableName, final ErrorReporter errors)
+      throws IOException {
     if (map == null) {
       map = new HashMap<String, Path>();
     }
+    for (HRegionInfo hri: mfs.getRegions(tableName)) {
+      if (errors != null) errors.progress();
 
-    // only include the directory paths to tables
-    Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
-    // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
-    // should be regions.
-    PathFilter familyFilter = new FamilyDirFilter(fs);
-    List<Path> regionDirs = FsLayout.getRegionDirPaths(fs, tableDir);
-    for (Path dd : regionDirs) {
-      if (null != errors) {
-        errors.progress();
-      }
-      // else its a region name, now look in region for families
-      FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
-      for (FileStatus familyDir : familyDirs) {
-        if (null != errors) {
-          errors.progress();
-        }
-        Path family = familyDir.getPath();
-        if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
-          continue;
-        }
-        // now in family, iterate over the StoreFiles and
-        // put in map
-        FileStatus[] familyStatus = fs.listStatus(family);
-        for (FileStatus sfStatus : familyStatus) {
-          if (null != errors) {
-            errors.progress();
-          }
-          Path sf = sfStatus.getPath();
-          map.put( sf.getName(), sf);
+      HRegionFileSystem rfs = HRegionFileSystem.open(conf, hri);
+      for (String family: rfs.getFamilies()) {
+        if (errors != null) errors.progress();
+
+        for (StoreFileInfo storeFile: rfs.getStoreFiles(family)) {
+          if (errors != null) errors.progress();
+
+          Path sf = storeFile.getPath();
+          map.put(sf.getName(), sf);
         }
       }
     }
     return map;
   }
 
-  public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
-    int result = 0;
-    try {
-      for (Path familyDir:getFamilyDirs(fs, p)){
-        result += getReferenceFilePaths(fs, familyDir).size();
-      }
-    } catch (IOException e) {
-      LOG.warn("Error Counting reference files.", e);
-    }
-    return result;
-  }
-
   /**
    * Runs through the HBase rootdir and creates a reverse lookup map for
    * table StoreFile names to the full Path.
@@ -1643,18 +1438,13 @@ public abstract class FSUtils {
    * @return Map keyed by StoreFile name with a value of the full Path.
    * @throws IOException When scanning the directory fails.
    */
-  public static Map<String, Path> getTableStoreFilePathMap(
-    final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors)
-  throws IOException {
-    Map<String, Path> map = new HashMap<String, Path>();
-
-    // if this method looks similar to 'getTableFragmentation' that is because
-    // it was borrowed from it.
-
-    // only include the directory paths to tables
-    for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
-      getTableStoreFilePathMap(map, fs, hbaseRootDir,
-          FSUtils.getTableName(tableDir), errors);
+  public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
+      final Path hbaseRootDir, ErrorReporter errors) throws IOException {
+    final Map<String, Path> map = new HashMap<String, Path>();
+    HMasterFileSystem mfs = HMasterFileSystem.open(fs, hbaseRootDir);
+    for (TableName tableName: mfs.getTables()) {
+      if (errors != null) errors.progress();
+      getTableStoreFilePathMap(map, mfs, tableName, errors);
     }
     return map;
   }
@@ -1875,74 +1665,34 @@ public abstract class FSUtils {
     FileSystem fs =  FileSystem.get(conf);
     Path rootPath = FSUtils.getRootDir(conf);
     long startTime = EnvironmentEdgeManager.currentTime();
-    Path queryPath;
-    // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
-    if (null == desiredTable) {
-      queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
-    } else {
-      queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
-    }
 
-    // reject all paths that are not appropriate
-    PathFilter pathFilter = new PathFilter() {
-      @Override
-      public boolean accept(Path path) {
-        // this is the region name; it may get some noise data
-        if (null == path) {
-          return false;
-        }
 
-        // no parent?
-        Path parent = path.getParent();
-        if (null == parent) {
-          return false;
-        }
-
-        String regionName = path.getName();
-        if (null == regionName) {
-          return false;
-        }
-
-        if (!regionName.toLowerCase().matches("[0-9a-f]+")) {
-          return false;
-        }
-        return true;
+    HMasterFileSystem mfs = HMasterFileSystem.open(conf);
+    List<HRegionInfo> hris;
+    if (desiredTable != null) {
+      hris = mfs.getRegions(TableName.valueOf(desiredTable));
+    } else {
+      hris = new ArrayList<HRegionInfo>();
+      for (TableName tableName: mfs.getTables()) {
+        hris.addAll(mfs.getRegions(tableName));
       }
-    };
-
-    FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
+    }
 
-    if (null == statusList) {
+    if (hris.isEmpty()) {
       return;
-    } else {
-      LOG.debug("Query Path: " + queryPath + " ; # list of files: " +
-          statusList.length);
     }
 
     // lower the number of threads in case we have very few expected regions
-    threadPoolSize = Math.min(threadPoolSize, statusList.length);
+    threadPoolSize = Math.min(threadPoolSize, hris.size());
 
     // run in multiple threads
     ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize,
         threadPoolSize, 60, TimeUnit.SECONDS,
-        new ArrayBlockingQueue<Runnable>(statusList.length));
+        new ArrayBlockingQueue<Runnable>(hris.size()));
     try {
       // ignore all file status items that are not of interest
-      for (FileStatus regionStatus : statusList) {
-        if (null == regionStatus) {
-          continue;
-        }
-
-        if (!regionStatus.isDirectory()) {
-          continue;
-        }
-
-        Path regionPath = regionStatus.getPath();
-        if (null == regionPath) {
-          continue;
-        }
-
-        tpe.execute(new FSRegionScanner(fs, regionPath,
+      for (HRegionInfo hri: hris) {
+        tpe.execute(new FSRegionScanner(conf, hri,
             regionToBestLocalityRSMapping, regionDegreeLocalityMapping));
       }
     } finally {
@@ -2010,7 +1760,7 @@ public abstract class FSUtils {
   /**
    * @param c
    * @return The DFSClient DFSHedgedReadMetrics instance or null if can't be found or not on hdfs.
-   * @throws IOException 
+   * @throws IOException
    */
   public static DFSHedgedReadMetrics getDFSHedgedReadMetrics(final Configuration c)
       throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
deleted file mode 100644
index efd8124..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.NavigableSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.fs.layout.FsLayout;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.wal.WALSplitter;
-
-/**
- * Utility methods for interacting with the hbase.root file system.
- */
-@InterfaceAudience.Private
-public final class FSVisitor {
-  private static final Log LOG = LogFactory.getLog(FSVisitor.class);
-
-  public interface RegionVisitor {
-    void region(final String region) throws IOException;
-  }
-
-  public interface StoreFileVisitor {
-    void storeFile(final String region, final String family, final String hfileName)
-       throws IOException;
-  }
-
-  public interface RecoveredEditsVisitor {
-    void recoveredEdits (final String region, final String logfile)
-      throws IOException;
-  }
-
-  public interface LogFileVisitor {
-    void logFile (final String server, final String logfile)
-      throws IOException;
-  }
-
-  private FSVisitor() {
-    // private constructor for utility class
-  }
-
-  /**
-   * Iterate over the table store files
-   *
-   * @param fs {@link FileSystem}
-   * @param tableDir {@link Path} to the table directory
-   * @param visitor callback object to get the store files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitRegions(final FileSystem fs, final Path tableDir,
-      final RegionVisitor visitor) throws IOException {
-    List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs));
-    if (regions == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No regions under directory:" + tableDir);
-      }
-      return;
-    }
-
-    for (FileStatus region: regions) {
-      visitor.region(region.getPath().getName());
-    }
-  }
-
-  /**
-   * Iterate over the table store files
-   *
-   * @param fs {@link FileSystem}
-   * @param tableDir {@link Path} to the table directory
-   * @param visitor callback object to get the store files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
-      final StoreFileVisitor visitor) throws IOException {
-    List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs));
-    if (regions == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No regions under directory:" + tableDir);
-      }
-      return;
-    }
-
-    for (FileStatus region: regions) {
-      visitRegionStoreFiles(fs, region.getPath(), visitor);
-    }
-  }
-
-  /**
-   * Iterate over the region store files
-   *
-   * @param fs {@link FileSystem}
-   * @param regionDir {@link Path} to the region directory
-   * @param visitor callback object to get the store files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
-      final StoreFileVisitor visitor) throws IOException {
-    FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
-    if (families == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No families under region directory:" + regionDir);
-      }
-      return;
-    }
-
-    PathFilter fileFilter = new FSUtils.FileFilter(fs);
-    for (FileStatus family: families) {
-      Path familyDir = family.getPath();
-      String familyName = familyDir.getName();
-
-      // get all the storeFiles in the family
-      FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
-      if (storeFiles == null) {
-        if (LOG.isTraceEnabled()) {
-          LOG.trace("No hfiles found for family: " + familyDir + ", skipping.");
-        }
-        continue;
-      }
-
-      for (FileStatus hfile: storeFiles) {
-        Path hfilePath = hfile.getPath();
-        visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
-      }
-    }
-  }
-
-  /**
-   * Iterate over each region in the table and inform about recovered.edits
-   *
-   * @param fs {@link FileSystem}
-   * @param tableDir {@link Path} to the table directory
-   * @param visitor callback object to get the recovered.edits files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir,
-      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
-    List<FileStatus> regions = FsLayout.getRegionDirFileStats(fs, tableDir, new FSUtils.RegionDirFilter(fs));
-    if (regions == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No recoveredEdits regions under directory:" + tableDir);
-      }
-      return;
-    }
-
-    for (FileStatus region: regions) {
-      visitRegionRecoveredEdits(fs, region.getPath(), visitor);
-    }
-  }
-
-  /**
-   * Iterate over recovered.edits of the specified region
-   *
-   * @param fs {@link FileSystem}
-   * @param regionDir {@link Path} to the Region directory
-   * @param visitor callback object to get the recovered.edits files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir,
-      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
-    NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regionDir);
-    if (files == null || files.size() == 0) return;
-
-    for (Path source: files) {
-      // check to see if the file is zero length, in which case we can skip it
-      FileStatus stat = fs.getFileStatus(source);
-      if (stat.getLen() <= 0) continue;
-
-      visitor.recoveredEdits(regionDir.getName(), source.getName());
-    }
-  }
-
-  /**
-   * Iterate over hbase log files
-   *
-   * @param fs {@link FileSystem}
-   * @param rootDir {@link Path} to the HBase root folder
-   * @param visitor callback object to get the log files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitLogFiles(final FileSystem fs, final Path rootDir,
-      final LogFileVisitor visitor) throws IOException {
-    Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
-    FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
-    if (logServerDirs == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No logs under directory:" + logsDir);
-      }
-      return;
-    }
-
-    for (FileStatus serverLogs: logServerDirs) {
-      String serverName = serverLogs.getPath().getName();
-
-      FileStatus[] wals = FSUtils.listStatus(fs, serverLogs.getPath());
-      if (wals == null) {
-        if (LOG.isTraceEnabled()) {
-          LOG.trace("No wals found for server: " + serverName + ", skipping.");
-        }
-        continue;
-      }
-
-      for (FileStatus walRef: wals) {
-        visitor.logFile(serverName, walRef.getPath().getName());
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 98ad22b..8b2796e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -111,6 +111,7 @@ import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.fs.layout.FsLayout;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
@@ -118,7 +119,6 @@ import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -395,7 +395,7 @@ public class HBaseFsck extends Configured implements Closeable {
           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
               + retryCounter.getMaxAttempts());
-          LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), 
+          LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
               ioe);
           try {
             exception = ioe;
@@ -762,7 +762,7 @@ public class HBaseFsck extends Configured implements Closeable {
         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
         // For each region, get the start and stop key from the META and compare them to the
         // same information from the Stores.
-        HRegionFileSystem hrfs = HRegionFileSystem.create(getConf(), fs, tableDir, regionInfo);
+        HRegionFileSystem hrfs = HRegionFileSystem.open(getConf(), regionInfo, true);
         Path path = hrfs.getRegionDir();
         FileStatus[] files = fs.listStatus(path);
         // For all the column families in this region...
@@ -2279,8 +2279,7 @@ public class HBaseFsck extends Configured implements Closeable {
                   LOG.warn(hri + " start and stop keys are in the range of " + region
                       + ". The region might not be cleaned up from hdfs when region " + region
                       + " split failed. Hence deleting from hdfs.");
-                  HRegionFileSystem.deleteAndArchiveRegionFromFileSystem(getConf(), fs,
-                    FsLayout.getTableDirFromRegionDir(regionDir), hri);
+                  HRegionFileSystem.destroy(getConf(), hri);
                   return;
                 }
               }
@@ -2688,10 +2687,10 @@ public class HBaseFsck extends Configured implements Closeable {
         }
         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
       }
-      
+
       return regionsFromMeta;
     }
-    
+
     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
       ErrorReporter errors;
 
@@ -4041,7 +4040,7 @@ public class HBaseFsck extends Configured implements Closeable {
     public synchronized Void call() throws IOException {
       try {
         // level 2: <HBASE_DIR>/<table>/*
-        List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir.getPath(), 
+        List<FileStatus> regionDirs = FsLayout.getRegionDirFileStats(fs, tableDir.getPath(),
           new FSUtils.RegionDirFilter(fs));
         if (regionDirs == null) {
           return null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
index 7f74d55..f2dbf52 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.io.FileLink;
 import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index ea704f8..1b173bb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.NoServerForRegionException;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -412,7 +412,7 @@ public class RegionSplitter {
    * Alternative getCurrentNrHRS which is no longer available.
    * @param connection
    * @return Rough count of regionservers out on cluster.
-   * @throws IOException 
+   * @throws IOException
    */
   private static int getRegionServerCount(final Connection connection) throws IOException {
     try (Admin admin = connection.getAdmin()) {
@@ -767,7 +767,7 @@ public class RegionSplitter {
    * @param conf
    * @param tableName
    * @return A Pair where first item is table dir and second is the split file.
-   * @throws IOException 
+   * @throws IOException
    */
   private static Pair<Path, Path> getTableDirAndSplitFile(final Configuration conf,
       final TableName tableName)
@@ -785,7 +785,7 @@ public class RegionSplitter {
       getTableDirAndSplitFile(connection.getConfiguration(), tableName);
     Path tableDir = tableDirAndSplitFile.getFirst();
     Path splitFile = tableDirAndSplitFile.getSecond();
- 
+
     FileSystem fs = tableDir.getFileSystem(connection.getConfiguration());
 
     // Using strings because (new byte[]{0}).equals(new byte[]{0}) == false

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
index 303ed60..9cbb135 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.fs.layout.FsLayout;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -43,10 +42,6 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
 import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter;
-import org.apache.hadoop.hbase.util.FSUtils.HFileFilter;
-import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
 
 /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
deleted file mode 100644
index 7aa9761..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.UUID;
-import java.util.Set;
-import java.util.HashSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.wal.WALSplitter;
-import org.apache.hadoop.hbase.fs.layout.FsLayout;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test {@link FSUtils}.
- */
-@Category({MiscTests.class, MediumTests.class})
-public class TestFSVisitor {
-  private static final Log LOG = LogFactory.getLog(TestFSVisitor.class);
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
-  private final String TABLE_NAME = "testtb";
-
-  private Set<String> tableFamilies;
-  private Set<String> tableRegions;
-  private Set<String> recoveredEdits;
-  private Set<String> tableHFiles;
-  private Set<String> regionServers;
-  private Set<String> serverLogs;
-
-  private FileSystem fs;
-  private Path tableDir;
-  private Path logsDir;
-  private Path rootDir;
-
-  @Before
-  public void setUp() throws Exception {
-    fs = FileSystem.get(TEST_UTIL.getConfiguration());
-    rootDir = TEST_UTIL.getDataTestDir("hbase");
-    logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
-
-    tableFamilies = new HashSet<String>();
-    tableRegions = new HashSet<String>();
-    recoveredEdits = new HashSet<String>();
-    tableHFiles = new HashSet<String>();
-    regionServers = new HashSet<String>();
-    serverLogs = new HashSet<String>();
-    tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
-    createRecoverEdits(tableDir, tableRegions, recoveredEdits);
-    createLogs(logsDir, regionServers, serverLogs);
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    fs.delete(rootDir, true);
-  }
-
-  @Test
-  public void testVisitStoreFiles() throws IOException {
-    final Set<String> regions = new HashSet<String>();
-    final Set<String> families = new HashSet<String>();
-    final Set<String> hfiles = new HashSet<String>();
-    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
-      public void storeFile(final String region, final String family, final String hfileName)
-          throws IOException {
-        regions.add(region);
-        families.add(family);
-        hfiles.add(hfileName);
-      }
-    });
-    assertEquals(tableRegions, regions);
-    assertEquals(tableFamilies, families);
-    assertEquals(tableHFiles, hfiles);
-  }
-
-  @Test
-  public void testVisitRecoveredEdits() throws IOException {
-    final Set<String> regions = new HashSet<String>();
-    final Set<String> edits = new HashSet<String>();
-    FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
-      public void recoveredEdits (final String region, final String logfile)
-          throws IOException {
-        regions.add(region);
-        edits.add(logfile);
-      }
-    });
-    assertEquals(tableRegions, regions);
-    assertEquals(recoveredEdits, edits);
-  }
-
-  @Test
-  public void testVisitLogFiles() throws IOException {
-    final Set<String> servers = new HashSet<String>();
-    final Set<String> logs = new HashSet<String>();
-    FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
-      public void logFile (final String server, final String logfile) throws IOException {
-        servers.add(server);
-        logs.add(logfile);
-      }
-    });
-    assertEquals(regionServers, servers);
-    assertEquals(serverLogs, logs);
-  }
-
-
-  /*
-   * |-testtb/
-   * |----f1d3ff8443297732862df21dc4e57262/
-   * |-------f1/
-   * |----------d0be84935ba84b66b1e866752ec5d663
-   * |----------9fc9d481718f4878b29aad0a597ecb94
-   * |-------f2/
-   * |----------4b0fe6068c564737946bcf4fd4ab8ae1
-   */
-  private Path createTableFiles(final Path rootDir, final String tableName,
-      final Set<String> tableRegions, final Set<String> tableFamilies,
-      final Set<String> tableHFiles) throws IOException {
-    Path tableDir = new Path(rootDir, tableName);
-    for (int r = 0; r < 10; ++r) {
-      String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
-      tableRegions.add(regionName);
-      Path regionDir = FsLayout.getRegionDir(tableDir, regionName);
-      for (int f = 0; f < 3; ++f) {
-        String familyName = "f" + f;
-        tableFamilies.add(familyName);
-        Path familyDir = new Path(regionDir, familyName);
-        fs.mkdirs(familyDir);
-        for (int h = 0; h < 5; ++h) {
-         String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
-         tableHFiles.add(hfileName);
-         fs.createNewFile(new Path(familyDir, hfileName));
-        }
-      }
-    }
-    return tableDir;
-  }
-
-  /*
-   * |-testtb/
-   * |----f1d3ff8443297732862df21dc4e57262/
-   * |-------recovered.edits/
-   * |----------0000001351969633479
-   * |----------0000001351969633481
-   */
-  private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
-      final Set<String> recoverEdits) throws IOException {
-    for (String region: tableRegions) {
-      Path regionEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(FsLayout.getRegionDir(tableDir, region));
-      long seqId = System.currentTimeMillis();
-      for (int i = 0; i < 3; ++i) {
-        String editName = String.format("%019d", seqId + i);
-        recoverEdits.add(editName);
-        FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
-        stream.write(Bytes.toBytes("test"));
-        stream.close();
-      }
-    }
-  }
-
-  /*
-   * Old style
-   * |-.logs/
-   * |----server5,5,1351969633508/
-   * |-------server5,5,1351969633508.0
-   * |----server6,6,1351969633512/
-   * |-------server6,6,1351969633512.0
-   * |-------server6,6,1351969633512.3
-   * New style
-   * |-.logs/
-   * |----server3,5,1351969633508/
-   * |-------server3,5,1351969633508.default.0
-   * |----server4,6,1351969633512/
-   * |-------server4,6,1351969633512.default.0
-   * |-------server4,6,1351969633512.some_provider.3
-   */
-  private void createLogs(final Path logDir, final Set<String> servers,
-      final Set<String> logs) throws IOException {
-    for (int s = 0; s < 7; ++s) {
-      String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
-      servers.add(server);
-      Path serverLogDir = new Path(logDir, server);
-      if (s % 2 == 0) {
-        if (s % 3 == 0) {
-          server += ".default";
-        } else {
-          server += "." + s;
-        }
-      }
-      fs.mkdirs(serverLogDir);
-      for (int i = 0; i < 5; ++i) {
-        String logfile = server + '.' + i;
-        logs.add(logfile);
-        FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
-        stream.write(Bytes.toBytes("test"));
-        stream.close();
-      }
-    }
-  }
-}


Mime
View raw message