hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r677483 - in /hadoop/hbase/trunk/src: java/org/apache/hadoop/hbase/client/ java/org/apache/hadoop/hbase/util/ test/org/apache/hadoop/hbase/util/ testdata/
Date Thu, 17 Jul 2008 01:29:35 GMT
Author: stack
Date: Wed Jul 16 18:29:34 2008
New Revision: 677483

URL: http://svn.apache.org/viewvc?rev=677483&view=rev
Log:
HBASE-62 Allow user add arbitrary key/value pairs to table and column descriptors; part 1
-- simplify migration before changing it

Added:
    hadoop/hbase/trunk/src/testdata/HADOOP-2478-testdata-v0.1.zip   (with props)
Removed:
    hadoop/hbase/trunk/src/testdata/HADOOP-2478-testdata.zip
Modified:
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=677483&r1=677482&r2=677483&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed Jul 16
18:29:34 2008
@@ -381,7 +381,6 @@
     }
 
     // Wait until all regions are disabled
-    
     while (isTableEnabled(tableName)) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Sleep. Waiting for all regions to be disabled from " +
@@ -389,7 +388,6 @@
       }
       try {
         Thread.sleep(pause);
-        
       } catch (InterruptedException e) {
         // continue
       }

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=677483&r1=677482&r2=677483&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java Wed Jul 16 18:29:34
2008
@@ -20,19 +20,12 @@
 
 package org.apache.hadoop.hbase.util;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -70,7 +63,7 @@
  * 
  * <p>A migration script must accompany any patch that changes data formats.
  * 
- * <p>This script has a 'check' and 'excecute' mode.  Adding migration steps,
+ * <p>This script has a 'check' and 'execute' mode.  Adding migration steps,
  * its important to keep this in mind.  Testing if migration needs to be run,
  * be careful not to make presumptions about the current state of the data in
  * the filesystem.  It may have a format from many versions previous with
@@ -79,6 +72,10 @@
  * old formats -- or, worse, fail in ways that are hard to figure (One such is
  * edits made by previous migration steps not being apparent in later migration
  * steps).  The upshot is always verify presumptions migrating.
+ * 
+ * <p>This script will migrate an hbase 0.1 install to a 0.2 install only.
+ * 
+ * @see <a href="http://wiki.apache.org/hadoop/Hbase/HowToMigrate">How To Migration</a>
  */
 public class Migrate extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(Migrate.class);
@@ -91,36 +88,13 @@
   // Gets set by migration methods if we are in readOnly mode.
   private boolean migrationNeeded = false;
 
-  /** Action to take when an extra file or unrecoverd log file is found */
-  private static String ACTIONS = "abort|ignore|delete|prompt";
-  private static enum ACTION  {
-    /** Stop conversion */
-    ABORT,
-    /** print a warning message, but otherwise ignore */
-    IGNORE,
-    /** delete extra files */
-    DELETE,
-    /** prompt for disposition of extra files */
-    PROMPT
-  }
-  
-  private static final Map<String, ACTION> options =
-    new HashMap<String, ACTION>();
-  
-  static {
-   options.put("abort", ACTION.ABORT);
-   options.put("ignore", ACTION.IGNORE);
-   options.put("delete", ACTION.DELETE);
-   options.put("prompt", ACTION.PROMPT);
-  }
-
   private boolean readOnly = false;
-  private ACTION otherFiles = ACTION.IGNORE;
-
-  private BufferedReader reader = null;
   
   private final Set<String> references = new HashSet<String>();
   
+  // Filesystem version of hbase 0.1.x.
+  private static final float HBASE_0_1_VERSION = 0.1f;
+  
   /** default constructor */
   public Migrate() {
     this(new HBaseConfiguration());
@@ -201,38 +175,23 @@
       LOG.info("Starting upgrade" + (readOnly ? " check" : ""));
 
       // See if there is a file system version file
-      String version = FSUtils.getVersion(fs, FSUtils.getRootDir(this.conf));
-      if (version != null && 
-          version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) {
+      String versionStr = FSUtils.getVersion(fs, FSUtils.getRootDir(this.conf));
+      if (versionStr != null &&
+          versionStr.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) {
         LOG.info("No upgrade necessary.");
         return 0;
       }
-
-      // Dependent on which which version hbase is at, run appropriate
-      // migrations.  Be consious that scripts can be run in readOnly -- i.e.
-      // check if migration is needed -- and then in actual migrate mode.  Be
-      // careful when writing your scripts that you do not make presumption
-      // about state of the FileSystem.  For example, in script that migrates
-      // between 2 and 3, it should not presume the layout is that of v2.  If
-      // readOnly mode, the pre-v2 scripts may not have been run yet.
-      if (version == null) {
-        FileStatus[] rootFiles = getRootDirFiles();
-        migrateFromNoVersion(rootFiles);
-        migrateToV2(rootFiles);
-        migrateToV3();
-        migrateToV4();
-      } else if (version.compareTo("0.1") == 0) {
-        migrateToV2(getRootDirFiles());
-        migrateToV3();
-        migrateToV4();
-      } else if (version.compareTo("2") == 0) {
-        migrateToV3();
-      } else if (version.compareTo("3") == 0) {
-        migrateToV4();
-      } else if (version.compareTo("4") == 0) {
-        // Nothing to do.
+      if (versionStr == null || Float.parseFloat(versionStr) < 0.1) {
+        throw new IOException("Install 0.1.x of hbase and run its " +
+          "migration first");
+      }
+      float version = Float.parseFloat(versionStr);
+      if (version == 0.1f) {
+        checkForUnrecoveredLogFiles(getRootDirFiles());
+        migrate();
       } else {
-        throw new IOException("Unrecognized version: " + version);
+        throw new IOException("Unrecognized or non-migratable version: " +
+          version);
       }
 
       if (!readOnly) {
@@ -247,57 +206,11 @@
     } catch (Exception e) {
       LOG.fatal("Upgrade" +  (readOnly ? " check" : "") + " failed", e);
       return -1;
-      
-    }
-  }
-
-  private void migrateFromNoVersion(FileStatus[] rootFiles) throws IOException {
-    LOG.info("No file system version found. Checking to see if hbase in " +
-      "Filesystem is at revision 0.1");
-
-    // Check to see if new root region dir exists
-    boolean newRootRegion = checkNewRootRegionDirExists();
-    if (this.readOnly &&  !newRootRegion) {
-      this.migrationNeeded = true;
-      return;
-    }
-
-    // Check for unrecovered region server log files
-    checkForUnrecoveredLogFiles(rootFiles);
-
-    // Check for "extra" files and for old upgradable regions
-    extraFiles(rootFiles);
-
-    if (!newRootRegion) {
-      // find root region
-      String rootRegion = OLD_PREFIX +
-        HRegionInfo.ROOT_REGIONINFO.getEncodedName();
-      if (!fs.exists(new Path(FSUtils.getRootDir(this.conf), rootRegion))) {
-        throw new IOException("Cannot find root region " + rootRegion);
-      } else if (readOnly) {
-        migrationNeeded = true;
-      } else {
-        migrateRegionDir(HConstants.ROOT_TABLE_NAME, rootRegion);
-        scanRootRegion();
-
-        // scan for left over regions
-        extraRegions();
-      }
     }
   }
   
-  private void migrateToV2(FileStatus[] rootFiles) throws IOException {
-    LOG.info("Checking to see if hbase in Filesystem is at version 2.");
-    checkForUnrecoveredLogFiles(rootFiles);
-  }
-
-  private void migrateToV3() throws IOException {
-    LOG.info("Checking to see if hbase in Filesystem is at version 3.");
+  private void migrate() throws IOException {
     addHistorianFamilyToMeta();
-  }
-  
-  private void migrateToV4() throws IOException {
-    LOG.info("Checking to see if hbase in Filesystem is at version 4.");
     updateBloomFilters();
   }
 
@@ -309,14 +222,6 @@
     }
     return stats;
   }
-  
-  private boolean checkNewRootRegionDirExists() throws IOException {
-    Path rootRegionDir =  HRegion.getRegionDir(FSUtils.getRootDir(this.conf),
-      HRegionInfo.ROOT_REGIONINFO);
-    boolean newRootRegion = fs.exists(rootRegionDir);
-    this.migrationNeeded = !newRootRegion;
-    return newRootRegion;
-  }
 
   private void checkForUnrecoveredLogFiles(FileStatus[] rootFiles)
   throws IOException {
@@ -332,63 +237,12 @@
           " unrecovered region server logs. Please uninstall this version of " +
           "HBase, re-install the previous version, start your cluster and " +
           "shut it down cleanly, so that all region server logs are recovered" +
-          " and deleted.");
-    }
-  }
-  
-  // Check for files that should not be there or should be migrated
-  private void extraFiles(FileStatus[] stats) throws IOException {
-    for (int i = 0; i < stats.length; i++) {
-      String name = stats[i].getPath().getName();
-      boolean newRootRegion = checkNewRootRegionDirExists();
-      if (name.startsWith(OLD_PREFIX)) {
-        if (!newRootRegion) {
-          // We need to migrate if the new root region directory doesn't exist
-          migrationNeeded = true;
-          String regionName = name.substring(OLD_PREFIX.length());
-          try {
-            Integer.parseInt(regionName);
-          } catch (NumberFormatException e) {
-            extraFile(otherFiles, "Old region format can not be upgraded: " +
-                name, stats[i].getPath());
-          }
-        } else {
-          // Since the new root region directory exists, we assume that this
-          // directory is not necessary
-          extraFile(otherFiles, "Old region directory found: " + name,
-              stats[i].getPath());
-        }
-      } else {
-        // File name does not start with "hregion_"
-        if (!newRootRegion) {
-          // new root region directory does not exist. This is an extra file
-          String message = "Unrecognized file " + name;
-          extraFile(otherFiles, message, stats[i].getPath());
-        }
-      }
+          " and deleted.  Or, if you are sure logs are vestige of old " +
+          "failures in hbase, remove them and then rerun the migration.  " +
+          "Here are the problem log files: " + unrecoveredLogs);
     }
   }
 
-  private void extraFile(ACTION action, String message, Path p)
-  throws IOException {
-    
-    if (action == ACTION.ABORT) {
-      throw new IOException(message + " aborting");
-    } else if (action == ACTION.IGNORE) {
-      LOG.info(message + " ignoring");
-    } else if (action == ACTION.DELETE) {
-      LOG.info(message + " deleting");
-      fs.delete(p, true);
-    } else {
-      // ACTION.PROMPT
-      String response = prompt(message + " delete? [y/n]");
-      if (response.startsWith("Y") || response.startsWith("y")) {
-        LOG.info(message + " deleting");
-        fs.delete(p, true);
-      }
-    }
-  }
-  
   void migrateRegionDir(final byte [] tableName, String oldPath)
   throws IOException {
     // Create directory where table will live
@@ -460,30 +314,6 @@
     }
   }
 
-  private void extraRegions() throws IOException {
-    Path rootdir = FSUtils.getRootDir(this.conf);
-    FileStatus[] stats = fs.listStatus(rootdir);
-    if (stats == null || stats.length == 0) {
-      throw new IOException("No files found under root directory " +
-          rootdir.toString());
-    }
-    for (int i = 0; i < stats.length; i++) {
-      String name = stats[i].getPath().getName();
-      if (name.startsWith(OLD_PREFIX)) {
-        String encodedName = name.substring(OLD_PREFIX.length());
-        String message;
-        if (references.contains(encodedName)) {
-          message =
-            "Region not in meta table but other regions reference it " + name;
-        } else {
-          message = 
-            "Region not in meta table and no other regions reference it " + name;
-        }
-        extraFile(otherFiles, message, stats[i].getPath());
-      }
-    }
-  }
-  
   private void addHistorianFamilyToMeta() throws IOException {
     if (this.migrationNeeded) {
       // Be careful. We cannot use MetAutils if current hbase in the
@@ -570,16 +400,8 @@
   @SuppressWarnings("static-access")
   private int parseArgs(String[] args) {
     Options opts = new Options();
-    Option extraFiles = OptionBuilder.withArgName(ACTIONS)
-    .hasArg()
-    .withDescription("disposition of 'extra' files: {abort|ignore|delete|prompt}")
-    .create("extrafiles");
-    
-    opts.addOption(extraFiles);
-    
     GenericOptionsParser parser =
       new GenericOptionsParser(this.getConf(), opts, args);
-    
     String[] remainingArgs = parser.getRemainingArgs();
     if (remainingArgs.length != 1) {
       usage();
@@ -591,23 +413,6 @@
       usage();
       return -1;
     }
-
-    if (readOnly) {
-      this.otherFiles = ACTION.IGNORE;
-
-    } else {
-      CommandLine commandLine = parser.getCommandLine();
-
-      ACTION action = null;
-      if (commandLine.hasOption("extrafiles")) {
-        action = options.get(commandLine.getOptionValue("extrafiles"));
-        if (action == null) {
-          usage();
-          return -1;
-        }
-        this.otherFiles = action;
-      }
-    }
     return 0;
   }
   
@@ -616,27 +421,11 @@
     System.err.println("  check                            perform upgrade checks only.");
     System.err.println("  upgrade                          perform upgrade checks and modify
hbase.\n");
     System.err.println("  Options are:");
-    System.err.println("    -extrafiles={abort|ignore|delete|prompt}");
-    System.err.println("                                   action to take if \"extra\" files
are found.\n");
     System.err.println("    -conf <configuration file>     specify an application configuration
file");
     System.err.println("    -D <property=value>            use value for given property");
     System.err.println("    -fs <local|namenode:port>      specify a namenode");
   }
-  
-  private synchronized String prompt(String prompt) {
-    System.out.print(prompt + " > ");
-    System.out.flush();
-    if (reader == null) {
-      reader = new BufferedReader(new InputStreamReader(System.in));
-    }
-    try {
-      return reader.readLine();
-      
-    } catch (IOException e) {
-      return null;
-    }
-  }
-  
+
   /**
    * Main program
    * 
@@ -652,4 +441,4 @@
     }
     System.exit(status);
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java?rev=677483&r1=677482&r2=677483&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java Wed Jul 16 18:29:34
2008
@@ -44,7 +44,7 @@
 import org.apache.hadoop.hbase.io.RowResult;
 
 /**
- * 
+ * Runs migration of filesystem from hbase 0.1 to 0.2.
  */
 public class TestMigrate extends HBaseTestCase {
   private static final Log LOG = LogFactory.getLog(TestMigrate.class);
@@ -60,8 +60,8 @@
   private static final int EXPECTED_COUNT = 17576;
 
   /**
+   * Test migration
    * @throws IOException 
-   * 
    */
   public void testUpgrade() throws IOException {
     MiniDFSCluster dfsCluster = null;
@@ -71,40 +71,19 @@
       this.conf.set(HConstants.HBASE_DIR, new Path(
         dfsCluster.getFileSystem().getHomeDirectory(), "hbase").toString());
       FileSystem dfs = dfsCluster.getFileSystem();
-      Path root = dfs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
-      dfs.mkdirs(root);
-
-      FileSystem localfs = FileSystem.getLocal(conf);
-      // Get path for zip file.  If running this test in eclipse, define
-      // the system property src.testdata for your test run.
-      String srcTestdata = System.getProperty("src.testdata");
-      if (srcTestdata == null) {
-        throw new NullPointerException("Define src.test system property");
-      }
-      Path data = new Path(srcTestdata, "HADOOP-2478-testdata.zip");
-      if (!localfs.exists(data)) {
-        throw new FileNotFoundException(data.toString());
-      }
-      FSDataInputStream hs = localfs.open(data);
-      ZipInputStream zip = new ZipInputStream(hs);
-      unzip(zip, dfs, root);
-      zip.close();
-      hs.close();
-      listPaths(dfs, root, root.toString().length() + 1);
+      Path rootDir =
+        dfs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
+      dfs.mkdirs(rootDir);
+      loadTestData(dfs, rootDir);
+      listPaths(dfs, rootDir, rootDir.toString().length() + 1);
       
       Migrate u = new Migrate(conf);
       u.run(new String[] {"check"});
-      listPaths(dfs, root, root.toString().length() + 1);
+      listPaths(dfs, rootDir, rootDir.toString().length() + 1);
       
       u = new Migrate(conf);
       u.run(new String[] {"upgrade"});
-      listPaths(dfs, root, root.toString().length() + 1);
-      
-      // Remove version file and try again
-      dfs.delete(new Path(root, HConstants.VERSION_FILE_NAME), false);
-      u = new Migrate(conf);
-      u.run(new String[] {"upgrade"});
-      listPaths(dfs, root, root.toString().length() + 1);
+      listPaths(dfs, rootDir, rootDir.toString().length() + 1);
       
       // Try again. No upgrade should be necessary
       u = new Migrate(conf);
@@ -120,6 +99,32 @@
       }
     }
   }
+  
+  /*
+   * Load up test data.
+   * @param dfs
+   * @param rootDir
+   * @throws IOException
+   */
+  private void loadTestData(final FileSystem dfs, final Path rootDir)
+  throws IOException {
+    FileSystem localfs = FileSystem.getLocal(conf);
+    // Get path for zip file.  If running this test in eclipse, define
+    // the system property src.testdata for your test run.
+    String srcTestdata = System.getProperty("src.testdata");
+    if (srcTestdata == null) {
+      throw new NullPointerException("Define src.test system property");
+    }
+    Path data = new Path(srcTestdata, "HADOOP-2478-testdata-v0.1.zip");
+    if (!localfs.exists(data)) {
+      throw new FileNotFoundException(data.toString());
+    }
+    FSDataInputStream hs = localfs.open(data);
+    ZipInputStream zip = new ZipInputStream(hs);
+    unzip(zip, dfs, rootDir);
+    zip.close();
+    hs.close();
+  }
 
   /*
    * Verify can read the migrated table.
@@ -218,14 +223,14 @@
     }
   }
 
-  private void unzip(ZipInputStream zip, FileSystem dfs, Path root)
+  private void unzip(ZipInputStream zip, FileSystem dfs, Path rootDir)
   throws IOException {
     ZipEntry e = null;
     while ((e = zip.getNextEntry()) != null)  {
       if (e.isDirectory()) {
-        dfs.mkdirs(new Path(root, e.getName()));
+        dfs.mkdirs(new Path(rootDir, e.getName()));
       } else {
-        FSDataOutputStream out = dfs.create(new Path(root, e.getName()));
+        FSDataOutputStream out = dfs.create(new Path(rootDir, e.getName()));
         byte[] buffer = new byte[4096];
         int len;
         do {
@@ -240,9 +245,9 @@
     }
   }
   
-  private void listPaths(FileSystem fs, Path dir, int rootdirlength)
+  private void listPaths(FileSystem filesystem, Path dir, int rootdirlength)
   throws IOException {
-    FileStatus[] stats = fs.listStatus(dir);
+    FileStatus[] stats = filesystem.listStatus(dir);
     if (stats == null || stats.length == 0) {
       return;
     }
@@ -250,7 +255,7 @@
       String path = stats[i].getPath().toString();
       if (stats[i].isDir()) {
         System.out.println("d " + path);
-        listPaths(fs, stats[i].getPath(), rootdirlength);
+        listPaths(filesystem, stats[i].getPath(), rootdirlength);
       } else {
         System.out.println("f " + path + " size=" + stats[i].getLen());
       }

Added: hadoop/hbase/trunk/src/testdata/HADOOP-2478-testdata-v0.1.zip
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/testdata/HADOOP-2478-testdata-v0.1.zip?rev=677483&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/hbase/trunk/src/testdata/HADOOP-2478-testdata-v0.1.zip
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream



Mime
View raw message