hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r794471 - in /hadoop/hbase/trunk/src: java/org/apache/hadoop/hbase/util/FSUtils.java java/org/apache/hadoop/hbase/util/Migrate.java test/data/hbase-0.19-two-small-tables.zip
Date Thu, 16 Jul 2009 00:13:44 GMT
Author: stack
Date: Thu Jul 16 00:13:44 2009
New Revision: 794471

URL: http://svn.apache.org/viewvc?rev=794471&view=rev
Log:
HBASE-1215 part2; check that all has been major compacted before starting migration

Modified:
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
    hadoop/hbase/trunk/src/test/data/hbase-0.19-two-small-tables.zip

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=794471&r1=794470&r2=794471&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java Thu Jul 16 00:13:44
2009
@@ -268,7 +268,6 @@
     return fs.exists(rootRegionDir);
   }
 
-
   /**
    * Runs through the hbase rootdir and checks all stores have only
    * one file in them -- that is, they've been major compacted.  Looks
@@ -312,6 +311,67 @@
   }
 
   /**
+   * Expects to find -ROOT- directory.
+   * @param fs
+   * @param hbaseRootDir
+   * @return True if this a pre020 layout.
+   * @throws IOException
+   */
+  public static boolean isPre020FileLayout(final FileSystem fs,
+    final Path hbaseRootDir)
+  throws IOException {
+    Path mapfiles = new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
+      "70236052"), "mapfiles");
+    return fs.exists(mapfiles);
+  }
+
+  /**
+   * Runs through the hbase rootdir and checks all stores have only
+   * one file in them -- that is, they've been major compacted.  Looks
+   * at root and meta tables too.  This version differs from
+   * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
+   * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
+   * @param fs
+   * @param hbaseRootDir
+   * @return True if this hbase install is major compacted.
+   * @throws IOException
+   */
+  public static boolean isMajorCompactedPre020(final FileSystem fs,
+      final Path hbaseRootDir)
+  throws IOException {
+    // Presumes any directory under hbase.rootdir is a table.
+    FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+    for (int i = 0; i < directories.length; i++) {
+      // Inside a table, there are compaction.dir directories to skip.
+      // Otherwise, all else should be regions.  Then in each region, should
+      // only be family directories.  Under each of these, should be a mapfile
+      // and info directory and in these only one file.
+      Path d = directories[i].getPath();
+      if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+      FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+      for (int j = 0; j < tablesubdirectories.length; j++) {
+        Path dd = tablesubdirectories[j].getPath();
+        if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+        // Else its a region name.  Now look in region for families.
+        FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+        for (int k = 0; k < familydirectories.length; k++) {
+          Path family = familydirectories[k].getPath();
+          // Now in family, there are 'mapfile' and 'info' subdirs.  Just
+          // look in the 'mapfile' subdir.
+          FileStatus [] familyStatus =
+            fs.listStatus(new Path(family, "mapfiles"));
+          if (familyStatus.length > 1) {
+            LOG.debug(family.toString() + " has " + familyStatus.length +
+              " files.");
+            return false;
+          }
+        }
+      }
+    }
+    return true;
+  }
+
+  /**
    * A {@link PathFilter} that returns directories.
    */
   public static class DirFilter implements PathFilter {

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=794471&r1=794470&r2=794471&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java Thu Jul 16 00:13:44
2009
@@ -26,10 +26,8 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -37,7 +35,6 @@
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
@@ -217,17 +214,20 @@
       return;
     }
     // Before we start, make sure all is major compacted.
-    if (!isMajorCompacted(fs, conf)) {
-      String msg = "All tables must be major compacted before the migration can begin." +
-        MIGRATION_LINK;
-      System.out.println(msg);
-      throw new IOException(msg);
+    Path hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    boolean pre020 = FSUtils.isPre020FileLayout(fs, hbaseRootDir);
+    if (pre020) {
+      if (!FSUtils.isMajorCompactedPre020(fs, hbaseRootDir)) {
+        String msg = "All tables must be major compacted before migration." +
+          MIGRATION_LINK;
+        System.out.println(msg);
+        throw new IOException(msg);
+      }
+      // TODO: Rewrite regions.
     }
+    // TOOD: Verify all has been brought over from old to new layout.
     final MetaUtils utils = new MetaUtils(this.conf);
     try {
-      // Preperation
-      // TODO: Fail if not all major compacted first
-      
       // TODO: Set the .META. and -ROOT- to flush at 16k?  32k?
       // TODO: Enable block cache on all tables
       // TODO: Rewrite MEMCACHE_FLUSHSIZE as MEMSTORE_FLUSHSIZE – name has changed.

@@ -270,33 +270,6 @@
     }
   }
 
-  /**
-   * Runs through the hbase rootdir and checks all stores have only
-   * one file in them -- that is, they've been major compacted.  Looks
-   * at root and meta tables too.
-   * @param fs
-   * @param c
-   * @return True if this hbase install is major compacted.
-   * @throws IOException
-   */
-  public static boolean isMajorCompacted(final FileSystem fs,
-      final HBaseConfiguration c)
-  throws IOException {
-    FileStatus [] directories =
-      fs.listStatus(new Path(c.get(HConstants.HBASE_DIR)), new PathFilter() {
-        public boolean accept(Path p) {
-          boolean isdir = false;
-          try {
-            isdir = fs.getFileStatus(p).isDir();
-          } catch (IOException e) {
-            e.printStackTrace();
-          }
-          return isdir;
-        }
-    });    
-    return true;
-  }
-
   /*
    * Enable blockcaching on catalog tables.
    * @param mr

Modified: hadoop/hbase/trunk/src/test/data/hbase-0.19-two-small-tables.zip
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/data/hbase-0.19-two-small-tables.zip?rev=794471&r1=794470&r2=794471&view=diff
==============================================================================
Binary files - no diff available.



Mime
View raw message