hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject svn commit: r613446 - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/util/ src/test/org/apache/hadoop/hbase/
Date Sat, 19 Jan 2008 20:20:16 GMT
Author: jimk
Date: Sat Jan 19 12:20:15 2008
New Revision: 613446

URL: http://svn.apache.org/viewvc?rev=613446&view=rev
Log:
HADOOP-2643 Make migration tool smarter.

Modified:
    lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
    lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
    lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java

Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Sat Jan 19 12:20:15 2008
@@ -201,7 +201,8 @@
                HMaster shutdown
    HADOOP-2616 hbase not spliting when the total size of region reaches max
                region size * 1.5
-
+   HADOOP-2643 Make migration tool smarter.
+   
 Release 0.15.1
 Branch 0.15
 

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
Sat Jan 19 12:20:15 2008
@@ -26,6 +26,14 @@
  */
 public interface HConstants {
   
+  // For migration
+
+  /** name of version file */
+  static final String VERSION_FILE_NAME = "hbase.version";
+  
+  /** version of file system */
+  static final String FILE_SYSTEM_VERSION = "0.1";
+  
   // Configuration parameters
   
   // TODO: URL for hbase master like hdfs URLs with host and port.

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Sat
Jan 19 12:20:15 2008
@@ -889,6 +889,10 @@
       // Make sure the root directory exists!
       if(! fs.exists(rootdir)) {
         fs.mkdirs(rootdir);
+        FSUtils.setVersion(fs, rootdir);
+      } else if (!FSUtils.checkVersion(fs, rootdir)) {
+        throw new IOException(
+            "file system not correct version. Run hbase.util.Migrate");
       }
 
       if (!fs.exists(rootRegionDir)) {

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
Sat Jan 19 12:20:15 2008
@@ -19,12 +19,16 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.DataInputStream;
 import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.dfs.DistributedFileSystem;
 
 /**
@@ -71,4 +75,40 @@
     }
     return available;
   }
+  
+  /**
+   * Verifies current version of file system
+   * 
+   * @param fs
+   * @param rootdir
+   * @return true if the current file system is the correct version
+   * @throws IOException
+   */
+  public static boolean checkVersion(FileSystem fs, Path rootdir) throws IOException {
+    Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
+    boolean versionOk = false;
+    if (fs.exists(versionFile)) {
+      FSDataInputStream s =
+        fs.open(new Path(rootdir, HConstants.VERSION_FILE_NAME));
+      String version = DataInputStream.readUTF(s);
+      s.close();
+      versionOk = version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0;
+    }
+    return versionOk;
+  }
+  
+  /**
+   * Sets version of file system
+   * 
+   * @param fs
+   * @param rootdir
+   * @throws IOException
+   */
+  public static void setVersion(FileSystem fs, Path rootdir) throws IOException {
+    FSDataOutputStream s =
+      fs.create(new Path(rootdir, HConstants.VERSION_FILE_NAME));
+    s.writeUTF(HConstants.FILE_SYSTEM_VERSION);
+    s.close();
+  }
+
 }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
Sat Jan 19 12:20:15 2008
@@ -136,7 +136,14 @@
     Path rootdir = fs.makeQualified(new Path(          // get path for instance
         conf.get(HConstants.HBASE_DIR, HConstants.DEFAULT_HBASE_DIR)));
 
-    // check for "extra" files
+    // See if there is a file system version file
+    
+    if (FSUtils.checkVersion(fs, rootdir)) {
+      LOG.info("file system is at current level, no upgrade necessary");
+      return 0;
+    }
+    
+    // check for "extra" files and for old upgradable regions
 
     extraFiles(fs, rootdir);
 
@@ -155,10 +162,14 @@
     // scan for left over regions
 
     extraRegions(fs, rootdir);
+    
+    // set file system version
+    
+    FSUtils.setVersion(fs, rootdir);
 
     return 0;
   }
-
+  
   private void extraFiles(FileSystem fs, Path rootdir) throws IOException {
     FileStatus[] stats = fs.listStatus(rootdir);
     if (stats == null || stats.length == 0) {
@@ -174,6 +185,15 @@
         } else {
           String message = "unrecognized file " + name;
           extraFile(otherFiles, message, fs, stats[i].getPath());
+        }
+      } else {
+        String regionName = name.substring(OLD_PREFIX.length());
+        try {
+          Integer.parseInt(regionName);
+          
+        } catch (NumberFormatException e) {
+          extraFile(otherFiles, "old region format can not be converted: " +
+              name, fs, stats[i].getPath());
         }
       }
     }

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Sat Jan 19 12:20:15 2008
@@ -28,6 +28,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
 
+import org.apache.hadoop.hbase.util.FSUtils;
+
 /**
  * This class creates a single process HBase cluster. One thread is created for
  * each server.
@@ -138,6 +140,7 @@
     try {
       this.parentdir = new Path(conf.get(HBASE_DIR, DEFAULT_HBASE_DIR));
       fs.mkdirs(parentdir);
+      FSUtils.setVersion(fs, parentdir);
       this.hbaseCluster = new LocalHBaseCluster(this.conf, nRegionNodes);
       this.hbaseCluster.startup();
     } catch(IOException e) {



Mime
View raw message