hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sra...@apache.org
Subject svn commit: r1344955 - in /hadoop/common/branches/branch-1-win: ./ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/hdfs/server/namenode/ src/test/org/apache/hadoop/hdfs/web/ src/test/org/apache/hadoop/security/
Date Fri, 01 Jun 2012 01:04:40 GMT
Author: sradia
Date: Fri Jun  1 01:04:40 2012
New Revision: 1344955

URL: http://svn.apache.org/viewvc?rev=1344955&view=rev
Log:
HADOOP-8411 TestStorageDirecotyFailure, TestTaskLogsTruncater, TestWebHdfsUrl and TestSecurityUtil
fail on Windows  (Ivan Mitic via Sanjay Radia)

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.txt
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Fri Jun  1 01:04:40 2012
@@ -20,6 +20,9 @@ branch-hadoop-1-win - unreleased
 
     HADOOP-8235 Support file permissions and ownership on Windows for RawLocalFileSystem
 (Chuan Liu via sanjay)
 
+    HADOOP-8411 TestStorageDirecotyFailure, TestTaskLogsTruncater, TestWebHdfsUrl and TestSecurityUtil
+                fail on Windows  (Ivan Mitic via Sanjay Radia)
+
 Release 1.1.0 - unreleased
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
Fri Jun  1 01:04:40 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.TaskLog.
 import org.apache.hadoop.mapred.TaskLog.LogFileDetail;
 import org.apache.hadoop.mapreduce.server.tasktracker.JVMInfo;
 import org.apache.hadoop.mapreduce.server.tasktracker.userlogs.UserLogManager;
+import org.apache.hadoop.util.Shell;
 
 /**
  * The class for truncating the user logs. 
@@ -284,7 +285,7 @@ public class TaskLogsTruncater {
       // ////// End of closing the file streams ////////////
 
       // ////// Commit the changes from tmp file to the logFile ////////////
-      if (!tmpFile.renameTo(logFile)) {
+      if (!renameAtomicWithOverride(tmpFile, logFile)) {
         // If the tmpFile cannot be renamed revert back
         // updatedTaskLogFileDetails to maintain the consistency of the
         // original log file
@@ -305,6 +306,43 @@ public class TaskLogsTruncater {
   }
 
   /**
+   * Renames a source file into a target file. Overrides the target file
+   * if it already exists.
+   */
+  private boolean renameAtomicWithOverride(File source, File target) {
+    if (Shell.WINDOWS && target.exists()) {
+      // Rename into the existing file fails on Windows, hence we have to
+      // rename using a temp file to provide some level of atomic behavior
+      // (and revert changes back if something fails)
+      File tmpTargetFile = new File(target.getPath() + ".tmp");
+      if (target.renameTo(tmpTargetFile)) {
+        if (source.renameTo(target)) {
+          // Rename succeeded, try to delete the target backup
+          if (!tmpTargetFile.delete()) {
+            LOG.warn("Cannot delete tmpTargetFile "
+                     + tmpTargetFile.getAbsolutePath());
+            tmpTargetFile.deleteOnExit();
+          }
+          // Return true anyways as we successfully renamed source
+          // into target
+          return true;
+        } else {
+          // Revert back the original
+          if (!tmpTargetFile.renameTo(target)) {
+            LOG.error("Cannot revert back the original log file "
+                     + target.getAbsolutePath());
+          }
+          return false;
+        }
+      } else {
+        return false;
+      }
+    } else {
+      return source.renameTo(target);
+    }
+  }
+
+  /**
    * @param lInfo
    * @param taskLogFileDetails
    * @param updatedTaskLogFileDetails

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
Fri Jun  1 01:04:40 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.util.Shell;
 
 /**
  * Test that the NN stays up as long as it has a valid storage directory and
@@ -125,6 +126,17 @@ public class TestStorageDirectoryFailure
     assertTrue(Arrays.equals(buff, readFile(name, buff.length)));
   }
 
+  /** Fully delete the storage directory */
+  private void FullyDeleteStorageDir(String name) throws IOException {
+    if (Shell.WINDOWS) {
+      // Files that are currently in use cannot be deleted on Windows, hence
+      // we are closing the log files explicitly.
+      cluster.getNameNode().getFSImage().getEditLog().close();
+      cluster.getNameNode().getFSImage().unlockAll();
+    }
+    FileUtil.fullyDelete(new File(name));
+  }
+
   @Test
   /** Remove storage dirs and checkpoint to trigger detection */
   public void testCheckpointAfterFailingFirstNamedir() throws IOException {
@@ -133,7 +145,7 @@ public class TestStorageDirectoryFailure
     checkFileCreation("file0");
 
     // Remove the 1st storage dir
-    FileUtil.fullyDelete(new File(nameDirs.get(0)));
+    FullyDeleteStorageDir(nameDirs.get(0));
     secondaryNN.doCheckpoint();
     assertEquals(1, numRemovedDirs());
     assertEquals(nameDirs.get(0), getRemovedDirs().get(0).getRoot().getPath());
@@ -141,7 +153,7 @@ public class TestStorageDirectoryFailure
     checkFileCreation("file1");
 
     // Remove the 2nd
-    FileUtil.fullyDelete(new File(nameDirs.get(1)));
+    FullyDeleteStorageDir(nameDirs.get(1));
     secondaryNN.doCheckpoint();
     assertEquals(2, numRemovedDirs());
     assertEquals(nameDirs.get(1), getRemovedDirs().get(1).getRoot().getPath());
@@ -156,7 +168,7 @@ public class TestStorageDirectoryFailure
 
     // After the checkpoint, we should be dead. Verify fatalExit was
     // called and that eg a checkpoint fails.
-    FileUtil.fullyDelete(new File(nameDirs.get(2)));
+    FullyDeleteStorageDir(nameDirs.get(2));
     try {
       secondaryNN.doCheckpoint();
       fail("There's no storage to retrieve an image from");
@@ -183,7 +195,7 @@ public class TestStorageDirectoryFailure
 
     checkFileCreation("file0");
 
-    FileUtil.fullyDelete(new File(nameDirs.get(0)));
+    FullyDeleteStorageDir(nameDirs.get(0));
     secondaryNN.doCheckpoint();
     assertEquals(1, numRemovedDirs());
     assertEquals(nameDirs.get(0), getRemovedDirs().get(0).getRoot().getPath());

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
Fri Jun  1 01:04:40 2012
@@ -71,14 +71,14 @@ public class TestWebHdfsUrl {
         fsPath, new TokenArgumentParam(tokenString));
     Assert.assertEquals(
         generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN,
-            ugi.getUserName())
+            ugi.getShortUserName())
             + "&token=" + tokenString, renewTokenUrl.getQuery());
     Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>(
         token);
     delegationToken.setKind(WebHdfsFileSystem.TOKEN_KIND);
     Assert.assertEquals(
         generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN,
-            ugi.getUserName())
+            ugi.getShortUserName())
             + "&token="
             + tokenString
             + "&"

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
Fri Jun  1 01:04:40 2012
@@ -95,7 +95,9 @@ public class TestSecurityUtil {
   
   @Test
   public void testLocalHostNameForNullOrWild() throws Exception {
-    String local = SecurityUtil.getLocalHostName();
+    // Get local host name and convert to lower case to be consistent with
+    // SecurityUtil.replacePattern
+    String local = SecurityUtil.getLocalHostName().toLowerCase();
     assertEquals("hdfs/" + local + "@REALM", SecurityUtil.getServerPrincipal(
         "hdfs/_HOST@REALM", (String) null));
     assertEquals("hdfs/" + local + "@REALM", SecurityUtil.getServerPrincipal(



Mime
View raw message