hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1486238 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Date Fri, 24 May 2013 22:20:56 GMT
Author: suresh
Date: Fri May 24 22:20:55 2013
New Revision: 1486238

URL: http://svn.apache.org/r1486238
Log:
HADOOP-8562. Merge r1477376 for HADOOP-9413, r1476877 for HDFS-4734, r1476856 for HADOOP-9524.

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1486238&r1=1486237&r2=1486238&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri May 24
22:20:55 2013
@@ -400,6 +400,9 @@ Release 2.0.5-beta - UNRELEASED
     HDFS-4705. Address HDFS test failures on Windows because of invalid
     dfs.namenode.name.dir. (Ivan Mitic via suresh)
 
+    HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows.
+    (Arpit Agarwal via suresh)
+
 Release 2.0.4-alpha - 2013-04-25
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1486238&r1=1486237&r2=1486238&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
Fri May 24 22:20:55 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.test.MockitoUtil;
+import org.apache.hadoop.util.Shell;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -73,6 +74,17 @@ public class TestDFSHAAdmin {
   private static String HOST_A = "1.2.3.1";
   private static String HOST_B = "1.2.3.2";
 
+  // Fencer shell commands that always return true and false respectively
+  // on Unix.
+  private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
+  private static String FENCER_FALSE_COMMAND_UNIX = "shell(false)";
+
+  // Fencer shell commands that always return true and false respectively
+  // on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
+  // commands 'rem' and 'help.exe'.
+  private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
+  private static String FENCER_FALSE_COMMAND_WINDOWS = "shell(help.exe /? >NUL)";
+
   private HdfsConfiguration getHAConf() {
     HdfsConfiguration conf = new HdfsConfiguration();
     conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);    
@@ -89,6 +101,16 @@ public class TestDFSHAAdmin {
     return conf;
   }
 
+  public static String getFencerTrueCommand() {
+    return Shell.WINDOWS ?
+        FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
+  }
+
+  public static String getFencerFalseCommand() {
+    return Shell.WINDOWS ?
+        FENCER_FALSE_COMMAND_WINDOWS : FENCER_FALSE_COMMAND_UNIX;
+  }
+
   @Before
   public void setup() throws IOException {
     mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
@@ -173,7 +195,7 @@ public class TestDFSHAAdmin {
     // Turn on auto-HA in the config
     HdfsConfiguration conf = getHAConf();
     conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
 
     // Should fail without the forcemanual flag
@@ -250,7 +272,7 @@ public class TestDFSHAAdmin {
   public void testFailoverWithFencerConfigured() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "nn1", "nn2"));
   }
@@ -259,7 +281,7 @@ public class TestDFSHAAdmin {
   public void testFailoverWithFencerAndNameservice() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2"));
   }
@@ -268,7 +290,7 @@ public class TestDFSHAAdmin {
   public void testFailoverWithFencerConfiguredAndForce() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
   }
@@ -277,7 +299,7 @@ public class TestDFSHAAdmin {
   public void testFailoverWithForceActive() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive"));
   }
@@ -286,7 +308,7 @@ public class TestDFSHAAdmin {
   public void testFailoverWithInvalidFenceArg() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence"));
   }
@@ -312,7 +334,7 @@ public class TestDFSHAAdmin {
     // Turn on auto-HA in the config
     HdfsConfiguration conf = getHAConf();
     conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
 
     assertEquals(0, runTool("-failover", "nn1", "nn2"));
@@ -323,7 +345,7 @@ public class TestDFSHAAdmin {
   public void testForceFenceOptionListedBeforeArgs() throws Exception {
     Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
     HdfsConfiguration conf = getHAConf();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
   }
@@ -359,23 +381,23 @@ public class TestDFSHAAdmin {
     
     HdfsConfiguration conf = getHAConf();
     // Set the default fencer to succeed
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
     
     // Set the NN-specific fencer to fail. Should fail to fence.
-    conf.set(nnSpecificKey, "shell(false)");
+    conf.set(nnSpecificKey, getFencerFalseCommand());
     tool.setConf(conf);
     assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
     conf.unset(nnSpecificKey);
 
     // Set an NS-specific fencer to fail. Should fail.
-    conf.set(nsSpecificKey, "shell(false)");
+    conf.set(nsSpecificKey, getFencerFalseCommand());
     tool.setConf(conf);
     assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
     
     // Set the NS-specific fencer to succeed. Should succeed
-    conf.set(nsSpecificKey, "shell(true)");
+    conf.set(nsSpecificKey, getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
   }

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java?rev=1486238&r1=1486237&r2=1486238&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Fri May 24 22:20:55 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.util.Shell;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.Before;
@@ -114,7 +115,8 @@ public class TestDFSHAAdminMiniCluster {
   
   @Test
   public void testTryFailoverToSafeMode() throws Exception {
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, 
+             TestDFSHAAdmin.getFencerTrueCommand());
     tool.setConf(conf);
 
     NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
@@ -136,10 +138,17 @@ public class TestDFSHAAdminMiniCluster {
     // tmp file, so we can verify that the args were substituted right
     File tmpFile = File.createTempFile("testFencer", ".txt");
     tmpFile.deleteOnExit();
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
-        "shell(echo -n $target_nameserviceid.$target_namenodeid " +
-        "$target_port $dfs_ha_namenode_id > " +
-        tmpFile.getAbsolutePath() + ")");
+    if (Shell.WINDOWS) {
+      conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+          "shell(echo %target_nameserviceid%.%target_namenodeid% " +
+              "%target_port% %dfs_ha_namenode_id% > " +
+              tmpFile.getAbsolutePath() + ")");
+    } else {
+      conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+          "shell(echo -n $target_nameserviceid.$target_namenodeid " +
+          "$target_port $dfs_ha_namenode_id > " +
+          tmpFile.getAbsolutePath() + ")");
+    }
 
     // Test failover with fencer
     tool.setConf(conf);
@@ -156,9 +165,11 @@ public class TestDFSHAAdminMiniCluster {
     assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
     
     // The fence script should run with the configuration from the target
-    // node, rather than the configuration from the fencing node
-    assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1",
-        Files.toString(tmpFile, Charsets.UTF_8));
+    // node, rather than the configuration from the fencing node. Strip
+    // out any trailing spaces and CR/LFs which may be present on Windows.
+    String fenceCommandOutput =Files.toString(tmpFile, Charsets.UTF_8).
+            replaceAll(" *[\r\n]+", "");
+    assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", fenceCommandOutput);
     tmpFile.delete();
     
     // Test failover with forceactive option
@@ -181,7 +192,8 @@ public class TestDFSHAAdminMiniCluster {
     assertFalse(tmpFile.exists());
 
     // Test failover with force fence listed before the other arguments
-    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, 
+             TestDFSHAAdmin.getFencerTrueCommand());
     tool.setConf(conf);
     assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
   }



Mime
View raw message