hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r525290 [3/3] - in /lucene/hadoop/trunk: ./ bin/ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/
Date Tue, 03 Apr 2007 21:39:27 GMT
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java?view=diff&rev=525290&r1=525289&r2=525290
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/FileUtil.java Tue Apr  3 14:39:25 2007
@@ -24,6 +24,7 @@
 import java.util.zip.ZipFile;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * A collection of file-processing util methods
@@ -318,6 +319,65 @@
     }
   }
   
+  /**
+   * Class for creating hardlinks.
+   * Supports Unix, Cygwin, WindXP.
+   *  
+   * @author Konstantin Shvachko
+   */
+  public static class HardLink { 
+    enum OSType {
+      OS_TYPE_UNIX, 
+      OS_TYPE_WINXP; 
+    }
+  
+    private static String[] hardLinkCommand;
+    
+    static {
+      switch( getOSType() ) {
+      case OS_TYPE_WINXP:
+        hardLinkCommand = new String[] {"fsutil","hardlink","create",null,null};
+        break;
+      case OS_TYPE_UNIX:
+      default:
+        hardLinkCommand = new String[] {"ln",null,null};
+      }
+    }
+
+    static OSType getOSType() {
+      String osName = System.getProperty("os.name");
+      if( osName.indexOf( "Windows") >= 0 && 
+          (osName.indexOf( "XpP") >= 0 || osName.indexOf( "2003") >= 0 ) )
+        return OSType.OS_TYPE_WINXP;
+      else
+        return OSType.OS_TYPE_UNIX;
+    }
+    
+    public static void createHardLink(File target, 
+                                      File linkName ) throws IOException {
+      int len = hardLinkCommand.length;
+      hardLinkCommand[len-2] = target.getCanonicalPath();
+      hardLinkCommand[len-1] = linkName.getCanonicalPath();
+      // execute shell command
+      Process process = Runtime.getRuntime().exec( hardLinkCommand );
+      try {
+        if (process.waitFor() != 0) {
+          String errMsg = new BufferedReader(new InputStreamReader(
+              process.getInputStream())).readLine();
+          if( errMsg == null )  errMsg = "";
+          String inpMsg = new BufferedReader(new InputStreamReader(
+              process.getErrorStream())).readLine();
+          if( inpMsg == null )  inpMsg = "";
+          throw new IOException( errMsg + inpMsg );
+        }
+      } catch (InterruptedException e) {
+        throw new IOException( StringUtils.stringifyException( e ));
+      } finally {
+        process.destroy();
+      }
+    }
+  }
+
   /**
    * Create a soft link between a src and destination
    * only on a local disk. HDFS does not support this

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFS.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFS.java?view=diff&rev=525290&r1=525289&r2=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFS.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFS.java Tue Apr  3 14:39:25 2007
@@ -219,8 +219,8 @@
 
     int nameNodePort = 9000 + testCycleNumber++; // ToDo: settable base port
     String nameNodeSocketAddr = "localhost:" + nameNodePort;
-    NameNode nameNodeDaemon = new NameNode(new File[] { new File(nameFSDir) },
-        "localhost", nameNodePort, conf);
+    conf.set("dfs.name.dir", nameFSDir);
+    NameNode nameNodeDaemon = new NameNode("localhost", nameNodePort, conf);
     DFSClient dfsClient = null;
     try {
       //

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFSNamespaceLogging.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFSNamespaceLogging.java?view=diff&rev=525290&r1=525289&r2=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFSNamespaceLogging.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/ClusterTestDFSNamespaceLogging.java Tue Apr  3 14:39:25 2007
@@ -352,8 +352,7 @@
 	
     NameNode.format(conf);
     
-    nameNodeDaemon = new NameNode(new File[] { new File(nameFSDir) },
-        "localhost", nameNodePort, conf);
+    nameNodeDaemon = new NameNode("localhost", nameNodePort, conf);
 
      //
       //        start DataNodes

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?view=diff&rev=525290&r1=525289&r2=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Tue Apr  3 14:39:25 2007
@@ -20,6 +20,7 @@
 import java.io.*;
 import java.net.*;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.net.NetworkTopology;
 
@@ -180,7 +181,7 @@
           int nDatanodes,
           boolean formatNamenode,
           String[] racks) throws IOException {
-    this(0, conf, nDatanodes, false, formatNamenode, racks);
+    this(0, conf, nDatanodes, formatNamenode, racks);
   }
   
   /**
@@ -254,6 +255,7 @@
    * @param dataNodeFirst should the datanode be brought up before the namenode?
    * @param formatNamenode should the namenode be formatted before starting up ?
    * @param racks array of strings indicating racks that each datanode is on
+   * @deprecated use {@link #MiniDFSCluster(Configuration, int, String[])}
    */
   public MiniDFSCluster(int namenodePort, 
                         Configuration conf,
@@ -261,7 +263,42 @@
                         boolean dataNodeFirst,
                         boolean formatNamenode,
                         String[] racks) throws IOException {
+    this(namenodePort, conf, nDatanodes, 
+        ! cannotStartDataNodeFirst(dataNodeFirst) &&  
+        formatNamenode, racks);
+  }
+
+  /**
+   * NameNode should be always started first.
+   * Data-nodes need to handshake with the name-node before they can start.
+   * 
+   * @param dataNodeFirst should the datanode be brought up before the namenode?
+   * @return false if dataNodeFirst is false
+   * @throws IOException if dataNodeFirst is true
+   * 
+   * @deprecated should be removed when dataNodeFirst is gone.
+   */
+  private static boolean cannotStartDataNodeFirst( boolean dataNodeFirst 
+                                                  ) throws IOException {
+    if( dataNodeFirst )
+      throw new IOException( "NameNode should be always started first." );
+    return false;
+  }
 
+  /**
+   * Create the config and start up the servers.  If either the rpc or info port is already 
+   * in use, we will try new ports.
+   * @param namenodePort suggestion for which rpc port to use.  caller should use 
+   *                     getNameNodePort() to get the actual port used.
+   * @param nDatanodes Number of datanodes   
+   * @param formatNamenode should the namenode be formatted before starting up ?
+   * @param racks array of strings indicating racks that each datanode is on
+   */
+  public MiniDFSCluster(int namenodePort, 
+                        Configuration conf,
+                        int nDatanodes,
+                        boolean formatNamenode,
+                        String[] racks) throws IOException {
     this.conf = conf;
     
     this.nDatanodes = nDatanodes;
@@ -279,18 +316,16 @@
     this.conf.setInt("dfs.safemode.extension", 0);
 
     // Create the NameNode
-    if (formatNamenode) { NameNode.format(conf); }
+    StartupOption startOpt = 
+      formatNamenode ? StartupOption.FORMAT : StartupOption.REGULAR;
+    conf.setObject( "dfs.namenode.startup", startOpt );
+    conf.setObject( "dfs.datanode.startup", startOpt );
     nameNode = new NameNodeRunner();
     nameNodeThread = new Thread(nameNode);
 
     //
     // Start the MiniDFSCluster
     //
-    
-    if (dataNodeFirst) {
-      startDataNodes(conf, racks, data_dir);
-    }
-    
     // Start the namenode and wait for it to be initialized
     nameNodeThread.start();
     while (!nameNode.isCrashed() && !nameNode.isInitialized()) {
@@ -310,9 +345,7 @@
     this.conf.set("fs.default.name", nnAddr.getHostName()+ ":" + Integer.toString(nameNodePort));
     
     // Start the datanodes
-    if (!dataNodeFirst) {
-      startDataNodes(conf, racks, data_dir);
-    }
+    startDataNodes(conf, racks, data_dir);
     
     while (!nameNode.isCrashed() && !nameNode.isUp()) {
       try {                                     // let daemons get started

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSFinalize.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSFinalize.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSFinalize.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSFinalize.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,119 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import java.io.IOException;
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * This test ensures the appropriate response from the system when 
+ * the system is finalized.
+ *
+ * @author Nigel Daley
+ */
+public class TestDFSFinalize extends TestCase {
+ 
+  private static final Log LOG = LogFactory.getLog(
+    "org.apache.hadoop.dfs.TestDFSFinalize");
+  Configuration conf;
+  private int testCounter = 0;
+  
+  /**
+   * Writes an INFO log message containing the parameters.
+   */
+  void log(String label, int numDirs) {
+    LOG.info("============================================================");
+    LOG.info("***TEST " + (testCounter++) + "*** " 
+      + label + ":"
+      + " numDirs="+numDirs);
+  }
+  
+  /**
+   * Verify that the current directory exists and that the previous directory
+   * does not exist.  Verify that current hasn't been modified by comparing 
+   * the checksum of all it's containing files with their original checksum.
+   * Note that we do not check that previous is removed on the DataNode
+   * because its removal is asynchronous therefore we have no reliable
+   * way to know when it will happen.  
+   */
+  void checkResult(String[] nameNodeDirs, String[] dataNodeDirs) throws IOException {
+    for (int i = 0; i < nameNodeDirs.length; i++) {
+      assertTrue(new File(nameNodeDirs[i],"current").isDirectory());
+      assertTrue(new File(nameNodeDirs[i],"current/VERSION").isFile());
+      assertTrue(new File(nameNodeDirs[i],"current/edits").isFile());
+      assertTrue(new File(nameNodeDirs[i],"current/fsimage").isFile());
+      assertTrue(new File(nameNodeDirs[i],"current/fstime").isFile());
+    }
+    for (int i = 0; i < dataNodeDirs.length; i++) {
+      assertEquals(
+        UpgradeUtilities.checksumContents(
+          DATA_NODE, new File(dataNodeDirs[i],"current")),
+        UpgradeUtilities.checksumMasterContents(DATA_NODE));
+    }
+    for (int i = 0; i < nameNodeDirs.length; i++) {
+      assertFalse(new File(nameNodeDirs[i],"previous").isDirectory());
+    }
+  }
+ 
+  /**
+   * This test attempts to finalize the NameNode and DataNode.
+   */
+  public void testFinalize() throws Exception {
+    File[] baseDirs;
+    UpgradeUtilities.initialize();
+    
+    for (int numDirs = 1; numDirs <= 2; numDirs++) {
+      conf = UpgradeUtilities.initializeStorageStateConf(numDirs);
+      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
+      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
+      
+      log("Finalize with existing previous dir",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.REGULAR,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
+      UpgradeUtilities.startCluster(DATA_NODE,StartupOption.REGULAR,conf);
+      UpgradeUtilities.finalizeCluster(conf);
+      checkResult(nameNodeDirs, dataNodeDirs);
+      
+      log("Finalize without existing previous dir",numDirs);
+      UpgradeUtilities.finalizeCluster(conf);
+      checkResult(nameNodeDirs, dataNodeDirs);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+    } // end numDir loop
+  }
+ 
+  public static void main(String[] args) throws Exception {
+    new TestDFSFinalize().testFinalize();
+  }
+  
+}
+
+

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSRollback.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSRollback.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSRollback.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSRollback.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,232 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import java.io.IOException;
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.fs.Path;
+
+/**
+* This test ensures the appropriate response (successful or failure) from
+* the system when the system is rolled back under various storage state and
+* version conditions.
+*
+* @author Nigel Daley
+*/
+public class TestDFSRollback extends TestCase {
+ 
+  private static final Log LOG = LogFactory.getLog(
+    "org.apache.hadoop.dfs.TestDFSRollback");
+  Configuration conf;
+  private int testCounter = 0;
+  
+  /**
+   * Writes an INFO log message containing the parameters.
+   */
+  void log(String label, int numDirs) {
+    LOG.info("============================================================");
+    LOG.info("***TEST " + (testCounter++) + "*** " 
+      + label + ":"
+      + " numDirs="+numDirs);
+  }
+  
+  /**
+   * Verify that the new current directory is the old previous.  
+   * It is assumed that the server has recovered and rolled back.
+   */
+  void checkResult(NodeType nodeType, String[] baseDirs) throws IOException {
+    switch (nodeType) {
+      case NAME_NODE:
+        for (int i = 0; i < baseDirs.length; i++) {
+          assertTrue(new File(baseDirs[i],"current").isDirectory());
+          assertTrue(new File(baseDirs[i],"current/VERSION").isFile());
+          assertTrue(new File(baseDirs[i],"current/edits").isFile());
+          assertTrue(new File(baseDirs[i],"current/fsimage").isFile());
+          assertTrue(new File(baseDirs[i],"current/fstime").isFile());
+        }
+        break;
+      case DATA_NODE:
+        for (int i = 0; i < baseDirs.length; i++) {
+          assertEquals(
+            UpgradeUtilities.checksumContents(
+              nodeType, new File(baseDirs[i],"current")),
+            UpgradeUtilities.checksumMasterContents(nodeType));
+        }
+        break;
+    }
+    for (int i = 0; i < baseDirs.length; i++) {
+      assertFalse(new File(baseDirs[i],"previous").isDirectory());
+    }
+  }
+ 
+  /**
+   * Starts the given nodeType with the given operation.  The remaining 
+   * parameters are used to verify the expected result.
+   * 
+   * @param nodeType must not be null
+   */
+  void runTest(NodeType nodeType, StartupOption operation, boolean shouldStart) 
+    throws Exception 
+  {
+    if (shouldStart) {
+      UpgradeUtilities.startCluster(nodeType, operation, conf);
+      UpgradeUtilities.stopCluster(nodeType);
+    } else {
+      try {
+        UpgradeUtilities.startCluster(nodeType, operation, conf); // should fail
+        throw new AssertionError("Cluster should have failed to start");
+      } catch (Exception expected) {
+        // expected
+        //expected.printStackTrace();
+        assertFalse(UpgradeUtilities.isNodeRunning(nodeType));
+      } finally {
+        UpgradeUtilities.stopCluster(nodeType);
+      }
+    }
+  }
+ 
+  /**
+   * This test attempts to rollback the NameNode and DataNode under
+   * a number of valid and invalid conditions.
+   */
+  public void testRollback() throws Exception {
+    File[] baseDirs;
+    UpgradeUtilities.initialize();
+    
+    for (int numDirs = 1; numDirs <= 2; numDirs++) {
+      conf = UpgradeUtilities.initializeStorageStateConf(numDirs);
+      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
+      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
+      
+      log("Normal NameNode rollback",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      runTest(NAME_NODE, StartupOption.ROLLBACK, true);
+      checkResult(NAME_NODE, nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("Normal DataNode rollback",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.ROLLBACK,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
+      runTest(DATA_NODE, StartupOption.ROLLBACK, true);
+      checkResult(DATA_NODE, dataNodeDirs);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+
+      log("NameNode rollback without existing previous dir",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      runTest(NAME_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("DataNode rollback without existing previous dir",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.UPGRADE,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      runTest(DATA_NODE, StartupOption.ROLLBACK, true);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+
+      log("DataNode rollback with future stored layout version in previous",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.ROLLBACK,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
+      UpgradeUtilities.createVersionFile(DATA_NODE,baseDirs,
+        new StorageInfo(Integer.MIN_VALUE,
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        UpgradeUtilities.getCurrentFsscTime()));
+      runTest(DATA_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+      
+      log("DataNode rollback with newer fsscTime in previous",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.ROLLBACK,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
+      UpgradeUtilities.createVersionFile(DATA_NODE,baseDirs,
+        new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        Long.MAX_VALUE));
+      runTest(DATA_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+
+      log("NameNode rollback with no edits file",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.remove(new File(f,"edits"));
+      }
+      runTest(NAME_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode rollback with no image file",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.remove(new File(f,"fsimage")); 
+      }
+      runTest(NAME_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode rollback with corrupt version file",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.corruptFile(new File(f,"VERSION")); 
+      }
+      runTest(NAME_NODE, StartupOption.ROLLBACK, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode rollback with old layout version in previous",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      UpgradeUtilities.createVersionFile(NAME_NODE,baseDirs,
+        new StorageInfo(1,
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        UpgradeUtilities.getCurrentFsscTime()));
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+    } // end numDir loop
+  }
+ 
+  public static void main(String[] args) throws Exception {
+    new TestDFSRollback().testRollback();
+  }
+  
+}
+
+

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStartupVersions.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStartupVersions.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStartupVersions.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStartupVersions.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * This test ensures the appropriate response (successful or failure) from 
+ * a Datanode when the system is started with differing version combinations. 
+ * 
+ * @author Nigel Daley
+ */
+public class TestDFSStartupVersions extends TestCase {
+  
+  private static final Log LOG = LogFactory.getLog(
+    "org.apache.hadoop.dfs.TestDFSStartupVersions");
+  private static Path TEST_ROOT_DIR = new Path(
+    System.getProperty("test.build.data","/tmp").toString().replace(' ', '+'));
+  
+  /**
+   * Writes an INFO log message containing the parameters.
+   */
+  void log(String label, NodeType nodeType, Integer testCase, StorageInfo version) {
+    String testCaseLine = "";
+    if (testCase != null) {
+      testCaseLine = " testCase="+testCase;
+    }
+    LOG.info("============================================================");
+    LOG.info("***TEST*** " + label + ":"
+      + testCaseLine
+      + " nodeType="+nodeType
+      + " layoutVersion="+version.getLayoutVersion()
+      + " namespaceID="+version.getNamespaceID()
+      + " fsscTime="+version.getCTime());
+  }
+  
+  /**
+   * Initialize the versions array.  This array stores all combinations 
+   * of cross product:
+   *  {oldLayoutVersion,currentLayoutVersion,futureLayoutVersion} X
+   *    {currentNamespaceId,incorrectNamespaceId} X
+   *      {pastFsscTime,currentFsscTime,futureFsscTime}
+   */
+  private StorageInfo[] initializeVersions() throws Exception {
+    int layoutVersionOld = -3;
+    int layoutVersionCur = UpgradeUtilities.getCurrentLayoutVersion();
+    int layoutVersionNew = Integer.MIN_VALUE;
+    int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID();
+    int namespaceIdOld = Integer.MIN_VALUE;
+    long fsscTimeOld = Long.MIN_VALUE;
+    long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime();
+    long fsscTimeNew = Long.MAX_VALUE;
+    
+    return new StorageInfo[] {
+      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeOld), // 0
+      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeCur), // 1
+      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeNew), // 2
+      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeOld), // 3
+      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeCur), // 4
+      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeNew), // 5
+      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeOld), // 6
+      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeCur), // 7
+      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeNew), // 8
+      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeOld), // 9
+      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeCur), // 10
+      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeNew), // 11
+      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeOld), // 12
+      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeCur), // 13
+      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeNew), // 14
+      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeOld), // 15
+      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeCur), // 16
+      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeNew), // 17
+    };
+  }
+  
+  /**
+   * Determines if the given Namenode version and Datanode version
+   * are compatible with each other. Compatibility in this case mean
+   * that the Namenode and Datanode will successfully start up and
+   * will work together. The rules for compatibility,
+   * taken from the DFS Upgrade Design, are as follows:
+   * <pre>
+   * 1. The data-node does regular startup (no matter which options 
+   *    it is started with) if
+   *       softwareLV == storedLV AND 
+   *       DataNode.FSSCTime == NameNode.FSSCTime
+   * 2. The data-node performs an upgrade if it is started without any 
+   *    options and
+   *       |softwareLV| > |storedLV| OR 
+   *       (softwareLV == storedLV AND
+   *        DataNode.FSSCTime < NameNode.FSSCTime)
+   * 3. NOT TESTED: The data-node rolls back if it is started with
+   *    the -rollback option and
+   *       |softwareLV| >= |previous.storedLV| AND 
+   *       DataNode.previous.FSSCTime <= NameNode.FSSCTime
+   * 4. In all other cases the startup fails.
+   * </pre>
+   */
+  boolean isVersionCompatible(StorageInfo namenodeVer, StorageInfo datanodeVer) {
+    // check #0
+    if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) {
+      LOG.info("namespaceIDs are not equal: isVersionCompatible=false");
+      return false;
+    }
+    // check #1
+    int softwareLV = FSConstants.LAYOUT_VERSION;  // will also be Namenode's LV
+    int storedLV = datanodeVer.getLayoutVersion();
+    if (softwareLV == storedLV &&  
+        datanodeVer.getCTime() == namenodeVer.getCTime()) 
+    {
+      LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true");
+      return true;
+    }
+    // check #2
+    long absSoftwareLV = Math.abs((long)softwareLV);
+    long absStoredLV = Math.abs((long)storedLV);
+    if (absSoftwareLV > absStoredLV ||
+        (softwareLV == storedLV &&
+         datanodeVer.getCTime() < namenodeVer.getCTime())) 
+    {
+      LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true");
+      return true;
+    }
+    // check #4
+    LOG.info("default case: isVersionCompatible=false");
+    return false;
+  }
+  
+  /**
+   * This test ensures the appropriate response (successful or failure) from 
+   * a Datanode when the system is started with differing version combinations. 
+   * <pre>
+   * For each 3-tuple in the cross product
+   *   ({oldLayoutVersion,currentLayoutVersion,futureLayoutVersion},
+   *    {currentNamespaceId,incorrectNamespaceId},
+   *    {pastFsscTime,currentFsscTime,futureFsscTime})
+   *      1. Startup Namenode with version file containing 
+   *         (currentLayoutVersion,currentNamespaceId,currentFsscTime)
+   *      2. Attempt to startup Datanode with version file containing 
+   *         this iterations version 3-tuple
+   * </pre>
+   */
+  public void testVersions() throws Exception {
+    UpgradeUtilities.initialize();
+    Configuration conf = UpgradeUtilities.initializeStorageStateConf(1);
+    StorageInfo[] versions = initializeVersions();
+    UpgradeUtilities.createStorageDirs(
+      NAME_NODE, conf.getStrings("dfs.name.dir"), "current");
+    UpgradeUtilities.startCluster(NAME_NODE,StartupOption.REGULAR,conf);
+    StorageInfo nameNodeVersion = new StorageInfo(
+      UpgradeUtilities.getCurrentLayoutVersion(),
+      UpgradeUtilities.getCurrentNamespaceID(),
+      UpgradeUtilities.getCurrentFsscTime());
+    log("NameNode version info",NAME_NODE,null,nameNodeVersion);
+    try {
+      for (int i = 0; i < versions.length; i++) {
+        File[] storage = UpgradeUtilities.createStorageDirs(
+          DATA_NODE, conf.getStrings("dfs.data.dir"), "current");
+        log("DataNode version info",DATA_NODE,i,versions[i]);
+        UpgradeUtilities.createVersionFile(DATA_NODE, storage, versions[i]);
+        try {
+          UpgradeUtilities.startCluster(DATA_NODE,StartupOption.REGULAR,conf);
+        } catch (Exception ignore) {
+          // Ignore.  The asserts below will check for problems.
+          // ignore.printStackTrace();
+        }
+        assertTrue(UpgradeUtilities.isNodeRunning(NAME_NODE));
+        assertEquals(isVersionCompatible(nameNodeVersion, versions[i]),
+          UpgradeUtilities.isNodeRunning(DATA_NODE));
+        UpgradeUtilities.stopCluster(DATA_NODE);
+      }
+    } finally {
+      UpgradeUtilities.stopCluster(null);
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    new TestDFSStartupVersions().testVersions();
+  }
+  
+}
+

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStorageStateRecovery.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStorageStateRecovery.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStorageStateRecovery.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSStorageStateRecovery.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,245 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import java.io.IOException;
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.fs.Path;
+
+/**
+* This test ensures the appropriate response (successful or failure) from
+* the system when the system is started under various storage state and
+* version conditions.
+*
+* @author Nigel Daley
+*/
+public class TestDFSStorageStateRecovery extends TestCase {
+ 
+  private static final Log LOG = LogFactory.getLog(
+    "org.apache.hadoop.dfs.TestDFSStorageStateRecovery");
+  Configuration conf;
+  private int testCounter = 0;
+  
+  /**
+   * The test case table.  Each row represents a test case.  This table is
+   * taken from the table in Apendix A of the HDFS Upgrade Test Plan
+   * (TestPlan-HdfsUpgrade.html) attached to
+   * http://issues.apache.org/jira/browse/HADOOP-702
+   * The column meanings are:
+   *  0) current directory exists
+   *  1) previous directory exists
+   *  2) previous.tmp directory exists
+   *  3) removed.tmp directory exists
+   *  4) node should recover and startup
+   *  5) current directory should exist after recovery but before startup
+   *  6) previous directory should exist after recovery but before startup
+   */
+  static boolean[][] testCases = new boolean[][] {
+    new boolean[] {true,  false, false, false, true,  true,  false}, // 1
+    new boolean[] {true,  true,  false, false, true,  true,  true }, // 2
+    new boolean[] {true,  false, true,  false, true,  true,  true }, // 3
+    new boolean[] {true,  true,  true,  true,  false, false, false }, // 4
+    new boolean[] {true,  true,  true,  false, false, false, false }, // 4
+    new boolean[] {false, true,  true,  true,  false, false, false }, // 4
+    new boolean[] {false, true,  true,  false, false, false, false }, // 4
+    new boolean[] {false, false, false, false, false, false, false }, // 5
+    new boolean[] {false, true,  false, false, false, false, false }, // 6
+    new boolean[] {false, false, true,  false, true,  true,  false}, // 7
+    new boolean[] {true,  false, false, true,  true,  true,  false}, // 8
+    new boolean[] {true,  true,  false, true,  false, false, false }, // 9
+    new boolean[] {true,  true,  true,  true,  false, false, false }, // 10
+    new boolean[] {true,  false, true,  true,  false, false, false }, // 10
+    new boolean[] {false, true,  true,  true,  false, false, false }, // 10
+    new boolean[] {false, false, true,  true,  false, false, false }, // 10
+    new boolean[] {false, false, false, true,  false, false, false }, // 11
+    new boolean[] {false, true,  false, true,  true,  true,  true }, // 12
+  };
+  
+  /**
+   * Writes an INFO log message containing the parameters. Only
+   * the first 4 elements of the state array are included in the message.
+   */
+  void log(String label, int numDirs, int testCaseNum, boolean[] state) {
+    LOG.info("============================================================");
+    LOG.info("***TEST " + (testCounter++) + "*** " 
+      + label + ":"
+      + " numDirs="+numDirs
+      + " testCase="+testCaseNum
+      + " current="+state[0]
+      + " previous="+state[1]
+      + " previous.tmp="+state[2]
+      + " removed.tmp="+state[3]);
+  }
+  
+  /**
+   * Sets up the storage directories for the given node type, either
+   * dfs.name.dir or dfs.data.dir. For each element in dfs.name.dir or
+   * dfs.data.dir, the subdirectories represented by the first four elements 
+   * of the <code>state</code> array will be created and populated.
+   * See UpgradeUtilities.createStorageDirs().
+   * 
+   * @param nodeType
+   *   the type of node that storage should be created for. Based on this
+   *   parameter either dfs.name.dir or dfs.data.dir is used from the global conf.
+   * @param state
+   *   a row from the testCases table which indicates which directories
+   *   to setup for the node
+   * @return file paths representing either dfs.name.dir or dfs.data.dir
+   *   directories
+   */
+  String[] createStorageState(NodeType nodeType, boolean[] state) throws Exception {
+    String[] baseDirs = (nodeType == NAME_NODE ?
+      conf.getStrings("dfs.name.dir") :
+      conf.getStrings("dfs.data.dir"));
+    UpgradeUtilities.createEmptyDirs(baseDirs);
+    if (state[0])  // current
+      UpgradeUtilities.createStorageDirs(nodeType, baseDirs, "current");
+    if (state[1])  // previous
+      UpgradeUtilities.createStorageDirs(nodeType, baseDirs, "previous");
+    if (state[2])  // previous.tmp
+      UpgradeUtilities.createStorageDirs(nodeType, baseDirs, "previous.tmp");
+    if (state[3])  // removed.tmp
+      UpgradeUtilities.createStorageDirs(nodeType, baseDirs, "removed.tmp");
+    return baseDirs;
+  }
+ 
+  /**
+   * Verify that the current and/or previous exist as indicated by 
+   * the method parameters.  If previous exists, verify that
+   * it hasn't been modified by comparing the checksum of all it's
+   * containing files with their original checksum.  It is assumed that
+   * the server has recovered.
+   */
+  void checkResult(NodeType nodeType, String[] baseDirs, 
+                   boolean currentShouldExist, boolean previousShouldExist) 
+                     throws IOException
+  {
+    switch (nodeType) {
+      case NAME_NODE:
+        if (currentShouldExist) {
+          for (int i = 0; i < baseDirs.length; i++) {
+            assertTrue(new File(baseDirs[i],"current").isDirectory());
+            assertTrue(new File(baseDirs[i],"current/VERSION").isFile());
+            assertTrue(new File(baseDirs[i],"current/edits").isFile());
+            assertTrue(new File(baseDirs[i],"current/fsimage").isFile());
+            assertTrue(new File(baseDirs[i],"current/fstime").isFile());
+          }
+        }
+        break;
+      case DATA_NODE:
+        if (currentShouldExist) {
+          for (int i = 0; i < baseDirs.length; i++) {
+            assertEquals(
+              UpgradeUtilities.checksumContents(
+                nodeType, new File(baseDirs[i],"current")),
+              UpgradeUtilities.checksumMasterContents(nodeType));
+          }
+        }
+        break;
+    }
+    if (previousShouldExist) {
+      for (int i = 0; i < baseDirs.length; i++) {
+        assertTrue(new File(baseDirs[i],"previous").isDirectory());
+        assertEquals(
+          UpgradeUtilities.checksumContents(
+            nodeType, new File(baseDirs[i],"previous")),
+          UpgradeUtilities.checksumMasterContents(nodeType));
+      }
+    }
+  }
+ 
+  /**
+   * Does a regular start of the given nodeType.
+   * 
+   * @param nodeType must not be null
+   * @param indicates whether or not the node should start
+   */
+  void runTest(NodeType nodeType, boolean shouldStart) throws Exception {
+    if (shouldStart) {
+      UpgradeUtilities.startCluster(nodeType, StartupOption.REGULAR, conf);
+      UpgradeUtilities.stopCluster(nodeType);
+    } else {
+      try {
+        UpgradeUtilities.startCluster(nodeType, StartupOption.REGULAR, conf); // should fail
+        throw new AssertionError("Cluster should have failed to start");
+      } catch (Exception expected) {
+        // expected
+        //expected.printStackTrace();
+        assertFalse(UpgradeUtilities.isNodeRunning(nodeType));
+      } finally {
+        UpgradeUtilities.stopCluster(nodeType);
+      }
+    }
+  }
+ 
+  /**
+   * This test iterates over the testCases table and attempts
+   * to startup the NameNode and DataNode normally.
+   */
+  public void testStorageStates() throws Exception {
+    String[] baseDirs;
+    UpgradeUtilities.initialize();
+
+    for (int numDirs = 1; numDirs <= 2; numDirs++) {
+      conf = UpgradeUtilities.initializeStorageStateConf(numDirs);
+      for (int i = 0; i < testCases.length; i++) {
+        boolean[] testCase = testCases[i];
+        boolean shouldRecover = testCase[4];
+        boolean curAfterRecover = testCase[5];
+        boolean prevAfterRecover = testCase[6];
+
+        log("NAME_NODE recovery",numDirs,i,testCase);
+        baseDirs = createStorageState(NAME_NODE, testCase);
+        runTest(NAME_NODE, shouldRecover);
+        if (shouldRecover) {
+          checkResult(NAME_NODE, baseDirs, curAfterRecover, prevAfterRecover);
+        }
+        
+        log("DATA_NODE recovery",numDirs,i,testCase);
+        createStorageState(NAME_NODE, new boolean[] {true,true,false,false});
+        UpgradeUtilities.startCluster(NAME_NODE,StartupOption.REGULAR,conf);
+        baseDirs = createStorageState(DATA_NODE, testCase);
+        if (!testCase[0] && !testCase[1] && !testCase[2] && !testCase[3]) {
+          // DataNode will create and format current if no directories exist
+          runTest(DATA_NODE, true);
+        } else {
+          runTest(DATA_NODE, shouldRecover);
+          if (shouldRecover) {
+            checkResult(DATA_NODE, baseDirs, curAfterRecover, prevAfterRecover);
+          }
+        }
+        UpgradeUtilities.stopCluster(null);
+      } // end testCases loop
+    } // end numDirs loop
+  }
+ 
+  public static void main(String[] args) throws Exception {
+    new TestDFSStorageStateRecovery().testStorageStates();
+  }
+  
+}
+
+

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgrade.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgrade.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgrade.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,230 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import java.io.IOException;
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.fs.Path;
+
+/**
+* This test ensures the appropriate response (successful or failure) from
+* the system when the system is upgraded under various storage state and
+* version conditions.
+*
+* @author Nigel Daley
+*/
+public class TestDFSUpgrade extends TestCase {
+ 
+  private static final Log LOG = LogFactory.getLog(
+    "org.apache.hadoop.dfs.TestDFSUpgrade");
+  Configuration conf;
+  private int testCounter = 0;
+  
+  /**
+   * Writes an INFO log message containing the parameters.
+   */
+  void log(String label, int numDirs) {
+    LOG.info("============================================================");
+    LOG.info("***TEST " + (testCounter++) + "*** " 
+      + label + ":"
+      + " numDirs="+numDirs);
+  }
+  
+  /**
+   * Verify that the current and previous directories exist.  Verify that 
+   * previous hasn't been modified by comparing the checksum of all it's
+   * containing files with their original checksum.  It is assumed that
+   * the server has recovered and upgraded.
+   */
+  void checkResult(NodeType nodeType, String[] baseDirs) throws IOException {
+    switch (nodeType) {
+      case NAME_NODE:
+        for (int i = 0; i < baseDirs.length; i++) {
+          assertTrue(new File(baseDirs[i],"current").isDirectory());
+          assertTrue(new File(baseDirs[i],"current/VERSION").isFile());
+          assertTrue(new File(baseDirs[i],"current/edits").isFile());
+          assertTrue(new File(baseDirs[i],"current/fsimage").isFile());
+          assertTrue(new File(baseDirs[i],"current/fstime").isFile());
+        }
+        break;
+      case DATA_NODE:
+        for (int i = 0; i < baseDirs.length; i++) {
+          assertEquals(
+            UpgradeUtilities.checksumContents(
+              nodeType, new File(baseDirs[i],"current")),
+            UpgradeUtilities.checksumMasterContents(nodeType));
+        }
+        break;
+    }
+    for (int i = 0; i < baseDirs.length; i++) {
+      assertTrue(new File(baseDirs[i],"previous").isDirectory());
+      assertEquals(
+        UpgradeUtilities.checksumContents(
+          nodeType, new File(baseDirs[i],"previous")),
+        UpgradeUtilities.checksumMasterContents(nodeType));
+    }
+  }
+ 
+  /**
+   * Starts the given nodeType with the given operation.  The remaining 
+   * parameters are used to verify the expected result.
+   * 
+   * @param nodeType must not be null
+   */
+  void runTest(NodeType nodeType, StartupOption operation, boolean shouldStart) 
+    throws Exception 
+  {
+    if (shouldStart) {
+      UpgradeUtilities.startCluster(nodeType, operation, conf);
+      UpgradeUtilities.stopCluster(nodeType);
+    } else {
+      try {
+        UpgradeUtilities.startCluster(nodeType, operation, conf); // should fail
+        throw new AssertionError("Cluster should have failed to start");
+      } catch (Exception expected) {
+        // expected
+        //expected.printStackTrace();
+        assertFalse(UpgradeUtilities.isNodeRunning(nodeType));
+      } finally {
+        UpgradeUtilities.stopCluster(nodeType);
+      }
+    }
+  }
+ 
+  /**
+   * This test attempts to upgrade the NameNode and DataNode under
+   * a number of valid and invalid conditions.
+   */
+  public void testUpgrade() throws Exception {
+    File[] baseDirs;
+    UpgradeUtilities.initialize();
+    
+    for (int numDirs = 1; numDirs <= 2; numDirs++) {
+      conf = UpgradeUtilities.initializeStorageStateConf(numDirs);
+      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
+      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
+      
+      log("Normal NameNode upgrade",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      runTest(NAME_NODE, StartupOption.UPGRADE, true);
+      checkResult(NAME_NODE, nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("Normal DataNode upgrade",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.UPGRADE,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      runTest(DATA_NODE, StartupOption.REGULAR, true);
+      checkResult(DATA_NODE, dataNodeDirs);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+      
+      log("NameNode upgrade with existing previous dir",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("DataNode upgrade with existing previous dir",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.UPGRADE,conf);
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
+      runTest(DATA_NODE, StartupOption.REGULAR, true);
+      checkResult(DATA_NODE, dataNodeDirs);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+
+      log("DataNode upgrade with future stored layout version in current",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.UPGRADE,conf);
+      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      UpgradeUtilities.createVersionFile(DATA_NODE,baseDirs,
+        new StorageInfo(Integer.MIN_VALUE,
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        UpgradeUtilities.getCurrentFsscTime()));
+      runTest(DATA_NODE, StartupOption.REGULAR, false);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+      
+      log("DataNode upgrade with newer fsscTime in current",numDirs);
+      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.startCluster(NAME_NODE,StartupOption.UPGRADE,conf);
+      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
+      UpgradeUtilities.createVersionFile(DATA_NODE,baseDirs,
+        new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        Long.MAX_VALUE));
+      runTest(DATA_NODE, StartupOption.REGULAR, false);
+      UpgradeUtilities.stopCluster(null);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
+
+      log("NameNode upgrade with no edits file",numDirs);
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.remove(new File(f,"edits"));
+      }
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode upgrade with no image file",numDirs);
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.remove(new File(f,"fsimage")); 
+      }
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode upgrade with corrupt version file",numDirs);
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      for (File f : baseDirs) { 
+        UpgradeUtilities.corruptFile(new File(f,"VERSION")); 
+      }
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+      
+      log("NameNode upgrade with future layout version in current",numDirs);
+      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
+      UpgradeUtilities.createVersionFile(NAME_NODE,baseDirs,
+        new StorageInfo(Integer.MIN_VALUE,
+                        UpgradeUtilities.getCurrentNamespaceID(),
+                        UpgradeUtilities.getCurrentFsscTime()));
+      runTest(NAME_NODE, StartupOption.UPGRADE, false);
+      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+    } // end numDir loop
+  }
+ 
+  public static void main(String[] args) throws Exception {
+    new TestDFSUpgrade().testUpgrade();
+  }
+  
+}
+
+

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCorruption.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCorruption.java?view=diff&rev=525290&r1=525289&r2=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCorruption.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestFileCorruption.java Tue Apr  3 14:39:25 2007
@@ -52,7 +52,7 @@
       util.createFiles(fs, "/srcdat");
       // Now deliberately remove the blocks
       File data_dir = new File(System.getProperty("test.build.data"),
-          "dfs/data/data5/data");
+          "dfs/data/data5/current");
       assertTrue("data directory does not exist", data_dir.exists());
       File[] blocks = data_dir.listFiles();
       assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/UpgradeUtilities.java?view=auto&rev=525290
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/UpgradeUtilities.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/UpgradeUtilities.java Tue Apr  3 14:39:25 2007
@@ -0,0 +1,574 @@
+/*
+ * UpgradeUtilities.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.InetSocketAddress;
+import java.util.Random;
+import java.util.zip.CRC32;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.dfs.FSConstants.NodeType;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
+import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.hadoop.dfs.Storage.StorageDirectory;
+
+/**
+ * This class defines a number of static helper methods used by the
+ * DFS Upgrade unit tests.  By default, a singleton master populated storage
+ * directory is created for a Namenode (contains edits, fsimage,
+ * version, and time files) and a Datanode (contains version and
+ * block files).  The master directories are lazily created.  They are then
+ * copied by the createStorageDirs() method to create new storage
+ * directories of the appropriate type (Namenode or Datanode).
+ * 
+ * @author Nigel Daley
+ */
+public class UpgradeUtilities {
+
+  // The fs.default.name configuration host:port value for the Namenode
+  private static final String NAMENODE_HOST = "localhost:0";
+  // Root scratch directory on local filesystem 
+  private static File TEST_ROOT_DIR = new File(
+    System.getProperty("test.build.data","/tmp").toString().replace(' ', '+'));
+  // The singleton master storage directory for Namenode
+  private static File namenodeStorage = new File(TEST_ROOT_DIR, "namenodeMaster");
+  // A checksum of the contents in namenodeStorage directory
+  private static long namenodeStorageChecksum;
+  // The namespaceId of the namenodeStorage directory
+  private static int namenodeStorageNamespaceID;
+  // The fsscTime of the namenodeStorage directory
+  private static long namenodeStorageFsscTime;
+  // The singleton master storage directory for Datanode
+  private static File datanodeStorage = new File(TEST_ROOT_DIR, "datanodeMaster");
+  // A checksum of the contents in datanodeStorage directory
+  private static long datanodeStorageChecksum;
+  // The NameNode started by this Utility class
+  private static NameNode namenode = null;
+  // The DataNode started by this Utility class
+  private static DataNode datanode = null;
+  
+  /**
+   * Initialize the data structures used by this class.  
+   * IMPORTANT NOTE: This method must be called once before calling 
+   *                 any other public method on this class.  
+   */
+  public static void initialize() throws Exception {
+    createEmptyDirs(new String[] {TEST_ROOT_DIR.toString()});
+    initializeStorage();
+  }
+  
+  /**
+   * Initialize dfs.name.dir and dfs.data.dir with the specified number of
+   * directory entries. Also initialize fs.default.name and 
+   * dfs.blockreport.intervalMsec.
+   */
+  public static Configuration initializeStorageStateConf(int numDirs) {
+    StringBuffer nameNodeDirs =
+      new StringBuffer(new File(TEST_ROOT_DIR, "name1").toString());
+    StringBuffer dataNodeDirs =
+      new StringBuffer(new File(TEST_ROOT_DIR, "data1").toString());
+    for (int i = 2; i <= numDirs; i++) {
+      nameNodeDirs.append("," + new File(TEST_ROOT_DIR, "name"+i));
+      dataNodeDirs.append("," + new File(TEST_ROOT_DIR, "data"+i));
+    }
+    Configuration conf = new Configuration();
+    conf.set("dfs.name.dir", nameNodeDirs.toString());
+    conf.set("dfs.data.dir", dataNodeDirs.toString());
+    conf.set("dfs.blockreport.intervalMsec", 10000);
+    return conf;
+  }
+  
+  /**
+   * Starts the given type of node or all nodes.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   *
+   * @param nodeType
+   *    The type of node to start.  If NAME_NODE, then one
+   *    Namenode is started.  If DATA_NODE, then one Datanode
+   *    is started.
+   * @param operation
+   *    The operation with which to startup the given type
+   *    node. FORMAT and null are treated as a REGULAR startup. If nodeType
+   *    if DATA_NODE, then UPGRADE is also treated as REGULAR.
+   * @param conf
+   *    The configuration to be used in starting the node.
+   *
+   * @throw IllegalStateException
+   *    If this method is called to start a
+   *    node that is already running.
+   */
+  public static void startCluster(NodeType nodeType, StartupOption operation, Configuration conf) throws Exception {
+    if (isNodeRunning(nodeType)) {
+      throw new IllegalStateException("Attempting to start "
+        + nodeType + " but it is already running");
+    }
+    if (nodeType == DATA_NODE && operation == StartupOption.UPGRADE) {
+      operation = StartupOption.REGULAR;
+    }
+    String[] args = (operation == null ||
+      operation == StartupOption.FORMAT ||
+      operation == StartupOption.REGULAR) ?
+        new String[] {} : new String[] {"-"+operation.toString()};
+    switch (nodeType) {
+      case NAME_NODE:
+        // Set up the right ports for the datanodes
+        conf.set("fs.default.name",NAMENODE_HOST);
+        namenode = NameNode.createNameNode(args, conf);
+        break;
+      case DATA_NODE:
+        if (namenode == null) {
+          throw new IllegalStateException("Attempting to start DATA_NODE "
+            + "but NAME_NODE is not running");
+        }
+        // Set up the right ports for the datanodes
+        InetSocketAddress nnAddr = namenode.getNameNodeAddress(); 
+        conf.set("fs.default.name", nnAddr.getHostName()+ ":" + nnAddr.getPort());
+        conf.setInt("dfs.info.port", 0);
+        conf.setInt("dfs.datanode.info.port", 0);
+        datanode = DataNode.createDataNode(args, conf);
+        break;
+    }
+  }
+  
+  /**
+   * Stops the given type of node or all nodes.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   *
+   * @param nodeType
+   *    The type of node to stop if it is running. If null, then both
+   *    Namenode and Datanodes are stopped if they are running.
+   */
+  public static void stopCluster(NodeType nodeType) {
+    if (nodeType == NAME_NODE || nodeType == null) {
+      if (namenode != null) {
+        namenode.stop();
+      }
+      namenode = null;
+    }
+    if (nodeType == DATA_NODE || nodeType == null) {
+      if (datanode != null) {
+        datanode.shutdown(); 
+      }
+      DataNode.shutdownAll();
+      datanode = null;
+    }
+  }
+  
+  /**
+   * If the Namenode is running, attempt to finalize a previous upgrade.
+   * When this method return, the NameNode should be finalized, but
+   * DataNodes may not be since that occurs asynchronously.
+   *
+   * @throw IllegalStateException if the Namenode is not running.
+   */
+  public static void finalizeCluster(Configuration conf) throws Exception {
+    if (! isNodeRunning(NAME_NODE)) {
+      throw new IllegalStateException("Attempting to finalize "
+        + "Namenode but it is not running");
+    }
+    new DFSAdmin().doMain(conf, new String[] {"-finalizeUpgrade"});
+  }
+  
+  /**
+   * Determines if the given node type is currently running.
+   * If the node type is DATA_NODE, then all started Datanodes
+   * must be running in-order for this method to return
+   * <code>true</code>.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   */
+  public static boolean isNodeRunning(NodeType nodeType) {
+    switch( nodeType ) {
+      case NAME_NODE:
+        return namenode != null;
+      case DATA_NODE:
+        return datanode != null;
+      default:
+        assert false : "Invalid node type: " + nodeType;
+    }
+    return false;
+  }
+  
+  /**
+   * Format the given directories.  This is equivalent to the Namenode
+   * formatting the given directories.  If a given directory already exists,
+   * it is first deleted; otherwise if it does not exist, it is first created.
+   *
+   * @throw IOException if unable to format one of the given dirs
+   */
+  public static void format(File... dirs) throws IOException {
+    String imageDirs = "";
+    for (int i = 0; i < dirs.length; i++) {
+      if( i == 0 )
+        imageDirs = dirs[i].getCanonicalPath();
+      else
+        imageDirs += "," + dirs[i].getCanonicalPath();
+    }
+    Configuration conf = new Configuration();
+    conf.set("dfs.name.dir", imageDirs);
+    NameNode.format(conf);
+  }
+  
+  /**
+   * Create empty directories.  If a specified directory already exists
+   * then it is first removed.
+   */
+  public static void createEmptyDirs(String[] dirs) {
+    for (String d : dirs) {
+      File dir = new File(d);
+      if (dir.exists()) {
+        remove(dir);
+      }
+      dir.mkdirs();
+    }
+  }
+  
+  /**
+   * Return the checksum for the singleton master storage directory
+   * of the given node type.
+   */
+  public static long checksumMasterContents(NodeType nodeType) throws IOException {
+    if (nodeType == NAME_NODE) {
+      return namenodeStorageChecksum;
+    } else {
+      return datanodeStorageChecksum;
+    }
+  }
+  
+  /**
+   * Compute the checksum of all the files in the specified directory.
+   * The contents of subdirectories are not included. This method provides
+   * an easy way to ensure equality between the contents of two directories.
+   *
+   * @param nodeType if DATA_NODE then any file named "VERSION" is ignored.
+   *    This is because this file file is changed every time
+   *    the Datanode is started.
+   * @param dir must be a directory. Subdirectories are ignored.
+   *
+   * @throw IllegalArgumentException if specified directory is not a directory
+   * @throw IOException if an IOException occurs while reading the files
+   * @return the computed checksum value
+   */
+  public static long checksumContents(NodeType nodeType, File dir) throws IOException {
+    if (!dir.isDirectory()) {
+      throw new IllegalArgumentException(
+        "Given argument is not a directory:" + dir);
+    }
+    File[] list = dir.listFiles();
+    CRC32 checksum = new CRC32();
+    for (int i = 0; i < list.length; i++) {
+      if (list[i].isFile()) {
+        // skip VERSION file for DataNodes
+        if (nodeType == DATA_NODE &&
+          list[i].getName().equals("VERSION")) 
+        {
+          continue; 
+        }
+        FileInputStream fis = new FileInputStream(list[i]);
+        byte[] buffer = new byte[1024];
+        int bytesRead;
+        while ((bytesRead = fis.read(buffer)) != -1) {
+          checksum.update(buffer,0,bytesRead);
+        }
+        fis.close();
+      }
+    }
+    return checksum.getValue();
+  }
+  
+  /**
+   * Simulate the <code>dfs.name.dir</code> or <code>dfs.data.dir</code>
+   * of a populated DFS filesystem.
+   *
+   * This method creates and populates the directory specified by
+   *  <code>parent/dirName</code>, for each parent directory.
+   * The contents of the new directories will be
+   * appropriate for the given node type.  If the directory does not
+   * exist, it will be created.  If the directory already exists, it
+   * will first be deleted.
+   *
+   * By default, a singleton master populated storage
+   * directory is created for a Namenode (contains edits, fsimage,
+   * version, and time files) and a Datanode (contains version and
+   * block files).  These directories are then
+   * copied by this method to create new storage
+   * directories of the appropriate type (Namenode or Datanode).
+   *
+   * @return the array of created directories
+   */
+  public static File[] createStorageDirs(NodeType nodeType, String[] parents, String dirName) throws Exception {
+    File[] retVal = new File[parents.length];
+    for (int i = 0; i < parents.length; i++) {
+      File newDir = new File(parents[i], dirName);
+      createEmptyDirs(new String[] {newDir.toString()});
+      populateDir(nodeType, newDir);
+      retVal[i] = newDir;
+    }
+    return retVal;
+  }
+  
+  /**
+   * Create a <code>version</code> file inside the specified parent
+   * directory.  If such a file already exists, it will be overwritten.
+   * The given version string will be written to the file as the layout
+   * version. If null, then the current layout version will be used.
+   * The parent and nodeType parameters must not be null.
+   *
+   * @param version
+   *
+   * @return the created version file
+   */
+  public static File[] createVersionFile(NodeType nodeType, File[] parent,
+    StorageInfo version) throws IOException 
+  {
+    if (version == null)
+      version = getCurrentNamespaceInfo();
+    Storage storage = null;
+    File[] versionFiles = new File[parent.length];
+    for (int i = 0; i < parent.length; i++) {
+      File versionFile = new File(parent[i], "VERSION");
+      remove(versionFile);
+      switch (nodeType) {
+        case NAME_NODE:
+          System.out.println("HERE");
+          storage = new FSImage( version );
+          break;
+        case DATA_NODE:
+                  System.out.println("HERE2");
+          storage = new DataStorage( version, "doNotCare" );
+          break;
+      }
+      StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile());
+      sd.write(versionFile);
+      versionFiles[i] = versionFile;
+    }
+    return versionFiles;
+  }
+  
+  /**
+   * Remove the specified file.  If the given file is a directory,
+   * then the directory and all its contents will be removed.
+   */
+  public static boolean remove(File file) {
+    try {
+      boolean retVal = FileUtil.fullyDelete(file);
+      return retVal;
+    } catch (IOException ioe) {
+      // this should never happen
+      throw new IllegalStateException(
+        "WHAT? FileUtil.fullyDelete threw and IOException?",ioe);
+    }
+  }
+  
+  /**
+   * Corrupt the specified file.  Some random bytes within the file
+   * will be changed to some random values.
+   *
+   * @throw IllegalArgumentException if the given file is not a file
+   * @throw IOException if an IOException occurs while reading or writing the file
+   */
+  public static void corruptFile(File file) throws IOException {
+    if (!file.isFile()) {
+      throw new IllegalArgumentException(
+        "Given argument is not a file:" + file);
+    }
+    RandomAccessFile raf = new RandomAccessFile(file,"rws");
+    Random random = new Random();
+    for (long i = 0; i < raf.length(); i++) {
+      raf.seek(i);
+      if (random.nextBoolean()) {
+        raf.writeByte(random.nextInt());
+      }
+    }
+    raf.close();
+  }
+  
+  /**
+   * Retrieve the current NamespaceInfo object from a running Namenode.
+   */
+  public static NamespaceInfo getCurrentNamespaceInfo() throws IOException {
+    if (isNodeRunning(NAME_NODE))
+      return namenode.versionRequest();
+    return null;
+  }
+  
+  /**
+   * Return the layout version inherent in the current version
+   * of the Namenode, whether it is running or not.
+   */
+  public static int getCurrentLayoutVersion() {
+    return FSConstants.LAYOUT_VERSION;
+  }
+  
+  /**
+   * Return the namespace ID inherent in the currently running
+   * Namenode.  If no Namenode is running, return the namespace ID of
+   * the master Namenode storage directory.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   */
+  public static int getCurrentNamespaceID() throws IOException {
+    if (isNodeRunning(NAME_NODE)) {
+      return namenode.versionRequest().getNamespaceID();
+    }
+    return namenodeStorageNamespaceID;
+  }
+  
+  /**
+   * Return the File System State Creation Timestamp (FSSCTime) inherent
+   * in the currently running Namenode.  If no Namenode is running,
+   * return the FSSCTime of the master Namenode storage directory.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   */
+  public static long getCurrentFsscTime() throws IOException {
+    if (isNodeRunning(NAME_NODE)) {
+      return namenode.versionRequest().getCTime();
+    }
+    return namenodeStorageFsscTime;
+  }
+  
+  /**********************************************************************
+   ********************* PRIVATE METHODS ********************************
+   *********************************************************************/
+  
+  /**
+   * Populates the given directory with valid version, edits, and fsimage
+   * files.  The version file will contain the current layout version.
+   *
+   * The UpgradeUtilities.initialize() method must be called once before
+   * calling this method.
+   *
+   * @throw IllegalArgumentException if dir does not already exist
+   */
+  private static void populateDir(NodeType nodeType, File dir) throws Exception {
+    if (!dir.exists()) {
+      throw new IllegalArgumentException(
+        "Given argument is not an existing directory:" + dir);
+    }
+    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
+    switch (nodeType) {
+      case NAME_NODE:
+        localFS.copyToLocalFile(
+          new Path(namenodeStorage.toString(), "current"),
+          new Path(dir.toString()),
+          false);
+        break;
+      case DATA_NODE:
+        localFS.copyToLocalFile(
+          new Path(datanodeStorage.toString(), "current"),
+          new Path(dir.toString()),
+          false);
+        break;
+    }
+  }
+  
+  static void writeFile(FileSystem fs,
+    Path path,
+    byte[] buffer,
+    int bufferSize ) throws IOException {
+    OutputStream out;
+    out = fs.create(path, true, bufferSize, (short) 1, 1024);
+    out.write( buffer, 0, bufferSize );
+    out.close();
+  }
+  
+  /**
+   * Creates a singleton master populated storage
+   * directory for a Namenode (contains edits, fsimage,
+   * version, and time files) and a Datanode (contains version and
+   * block files).  This can be a lengthy operation.
+   *
+   * @param conf must not be null.  These properties will be set:
+   *    fs.default.name
+   *    dfs.name.dir
+   *    dfs.data.dir
+   */
+  private static void initializeStorage() throws Exception {
+    Configuration config = new Configuration();
+    config.set("fs.default.name",NAMENODE_HOST);
+    config.set("dfs.name.dir", namenodeStorage.toString());
+    config.set("dfs.data.dir", datanodeStorage.toString());
+
+    try {
+      // format data-node
+      createEmptyDirs(new String[] {datanodeStorage.toString()});
+
+      // format name-node
+      NameNode.format(config);
+      
+      // start name-node
+      startCluster(NAME_NODE, null, config);
+      namenodeStorageNamespaceID = namenode.versionRequest().getNamespaceID();
+      namenodeStorageFsscTime = namenode.versionRequest().getCTime();
+      
+      // start data-node
+      startCluster(DATA_NODE, null, config);
+      
+      FileSystem fs = FileSystem.get(config);
+      Path baseDir = new Path("/TestUpgrade");
+      fs.mkdirs( baseDir );
+      
+      // write some files
+      int bufferSize = 4096;
+      byte[] buffer = new byte[bufferSize];
+      for( int i=0; i < bufferSize; i++ )
+        buffer[i] = (byte)('0' + i % 50);
+      writeFile(fs, new Path(baseDir, "file1"), buffer, bufferSize);
+      writeFile(fs, new Path(baseDir, "file2"), buffer, bufferSize);
+      
+      // save image
+      namenode.getFSImage().saveFSImage();
+      namenode.getFSImage().getEditLog().open();
+      
+      // write more files
+      writeFile(fs, new Path(baseDir, "file3"), buffer, bufferSize);
+      writeFile(fs, new Path(baseDir, "file4"), buffer, bufferSize);
+    } finally {
+      // shutdown
+      stopCluster(null);
+      remove(new File(namenodeStorage,"in_use.lock"));
+      remove(new File(datanodeStorage,"in_use.lock"));
+    }
+    namenodeStorageChecksum = checksumContents(
+      NAME_NODE, new File(namenodeStorage,"current"));
+    datanodeStorageChecksum = checksumContents(
+      DATA_NODE, new File(datanodeStorage,"current"));
+  }
+
+}
+



Mime
View raw message