hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r771635 [1/4] - in /hadoop/core/trunk: ./ src/test/org/apache/hadoop/cli/ src/test/org/apache/hadoop/cli/util/ src/test/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 05 May 2009 09:16:52 GMT
Author: sharad
Date: Tue May  5 09:16:51 2009
New Revision: 771635

URL: http://svn.apache.org/viewvc?rev=771635&view=rev
Log:
HADOOP-5081. Split TestCLI into HDFS, Mapred and Core tests.

Added:
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/testHDFSConf.xml
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/testMRConf.xml
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/build.xml
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml
    hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/CommandExecutor.java
    hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May  5 09:16:51 2009
@@ -312,6 +312,8 @@
     number of fields to be sufficient to construct the select query. (Enis
     Soztutar via cdouglas)
 
+    HADOOP-5081. Split TestCLI into HDFS, Mapred and Core tests. (sharad)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

Modified: hadoop/core/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/build.xml?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/build.xml (original)
+++ hadoop/core/trunk/build.xml Tue May  5 09:16:51 2009
@@ -697,6 +697,8 @@
     <copy file="${test.src.dir}/org/apache/hadoop/hdfs/hadoop-14-dfs-dir.tgz" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/hdfs/hadoop-dfs-dir.txt" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/testConf.xml" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/org/apache/hadoop/cli/testHDFSConf.xml" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/org/apache/hadoop/cli/testMRConf.xml" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data15bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java Tue May  5 09:16:51 2009
@@ -34,12 +34,8 @@
 import org.apache.hadoop.cli.util.ComparatorData;
 import org.apache.hadoop.cli.util.CLITestData.TestCmd;
 import org.apache.hadoop.cli.util.CLITestData.TestCmd.CommandType;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.security.authorize.HadoopPolicyProvider;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
@@ -68,29 +64,22 @@
   
   //By default, run the tests. The other mode is to run the commands and not
   // compare the output
-  public static String testMode = TESTMODE_TEST;
+  protected String testMode = TESTMODE_TEST;
   
   // Storage for tests read in from the config file
-  static ArrayList<CLITestData> testsFromConfigFile = null;
-  static ArrayList<ComparatorData> testComparators = null;
-  static String testConfigFile = "testConf.xml";
-  String thisTestCaseName = null;
-  static ComparatorData comparatorData = null;
-  
-  private static Configuration conf = null;
-  private static MiniDFSCluster dfsCluster = null;
-  private static DistributedFileSystem dfs = null;
-  private static MiniMRCluster mrCluster = null;
-  private static String namenode = null;
-  private static String jobtracker = null;
-  private static String clitestDataDir = null;
-  private static String username = null;
+  protected ArrayList<CLITestData> testsFromConfigFile = null;
+  protected ArrayList<ComparatorData> testComparators = null;
+  protected String thisTestCaseName = null;
+  protected ComparatorData comparatorData = null;
+  protected Configuration conf = null;
+  protected String clitestDataDir = null;
+  protected String username = null;
   
   /**
    * Read the test config file - testConfig.xml
    */
-  private void readTestConfigFile() {
-    
+  protected void readTestConfigFile() {
+    String testConfigFile = getTestFile();
     if (testsFromConfigFile == null) {
       boolean success = false;
       testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
@@ -106,6 +95,10 @@
     }
   }
   
+  protected String getTestFile() {
+    return "testConf.xml";
+  }
+  
   /*
    * Setup
    */
@@ -113,60 +106,20 @@
     // Read the testConfig.xml file
     readTestConfigFile();
     
-    // Start up the mini dfs cluster
-    boolean success = false;
     conf = new Configuration();
     conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG,
                   HadoopPolicyProvider.class, PolicyProvider.class);
     conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, 
                     true);
 
-    // Many of the tests expect a replication value of 1 in the output
-    conf.setInt("dfs.replication", 1);
-    
-    // Build racks and hosts configuration to test dfsAdmin -printTopology
-    String [] racks =  {"/rack1", "/rack1", "/rack2", "/rack2",
-                        "/rack2", "/rack3", "/rack4", "/rack4" };
-    String [] hosts = {"host1", "host2", "host3", "host4",
-                       "host5", "host6", "host7", "host8" };
-    
-    dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
-    
-    namenode = conf.get("fs.default.name", "file:///");
     clitestDataDir = new File(TEST_CACHE_DATA_DIR).
-      toURI().toString().replace(' ', '+');
-    username = System.getProperty("user.name");
-
-    FileSystem fs = dfsCluster.getFileSystem();
-    assertTrue("Not a HDFS: "+fs.getUri(),
-               fs instanceof DistributedFileSystem);
-    dfs = (DistributedFileSystem) fs;
-    
-     // Start up mini mr cluster
-    JobConf mrConf = new JobConf(conf);
-    mrCluster = new MiniMRCluster(1, dfsCluster.getFileSystem().getUri().toString(), 1, 
-                           null, null, mrConf);
-    jobtracker = mrCluster.createJobConf().get("mapred.job.tracker", "local");
-
-    success = true;
-
-    assertTrue("Error setting up Mini DFS & MR clusters", success);
+    toURI().toString().replace(' ', '+');
   }
   
   /**
    * Tear down
    */
   public void tearDown() throws Exception {
-    boolean success = false;
-    mrCluster.shutdown();
-    
-    dfs.close();
-    dfsCluster.shutdown();
-    success = true;
-    Thread.sleep(2000);
-
-    assertTrue("Error tearing down Mini DFS & MR clusters", success);
-    
     displayResults();
   }
   
@@ -175,10 +128,8 @@
    * @param cmd
    * @return String expanded command
    */
-  private String expandCommand(final String cmd) {
+  protected String expandCommand(final String cmd) {
     String expCmd = cmd;
-    expCmd = expCmd.replaceAll("NAMENODE", namenode);
-    expCmd = expCmd.replaceAll("JOBTRACKER", jobtracker);
     expCmd = expCmd.replaceAll("CLITEST_DATA", clitestDataDir);
     expCmd = expCmd.replaceAll("USERNAME", username);
     
@@ -259,10 +210,18 @@
     LOG.info("");
     LOG.info("             Overall result: " + 
     		(overallResults ? "+++ PASS +++" : "--- FAIL ---"));
-    LOG.info("               # Tests pass: " + totalPass +
-    		" (" + (100 * totalPass / (totalPass + totalFail)) + "%)");
-    LOG.info("               # Tests fail: " + totalFail + 
-    		" (" + (100 * totalFail / (totalPass + totalFail)) + "%)");
+    if ((totalPass + totalFail) == 0) {
+      LOG.info("               # Tests pass: " + 0);
+      LOG.info("               # Tests fail: " + 0);
+    }
+    else 
+    {
+      LOG.info("               # Tests pass: " + totalPass +
+          " (" + (100 * totalPass / (totalPass + totalFail)) + "%)");
+      LOG.info("               # Tests fail: " + totalFail + 
+          " (" + (100 * totalFail / (totalPass + totalFail)) + "%)");
+    }
+    
     LOG.info("         # Validations done: " + totalComparators + 
     		" (each test may do multiple validations)");
     
@@ -310,7 +269,7 @@
    * @param compdata
    * @return
    */
-  private boolean compareTestOutput(ComparatorData compdata) {
+  private boolean compareTestOutput(ComparatorData compdata, Result cmdResult) {
     // Compare the output based on the comparator
     String comparatorType = compdata.getComparatorType();
     Class<?> comparatorClass = null;
@@ -327,7 +286,7 @@
         comparatorClass = Class.forName("org.apache.hadoop.cli.util." + 
           comparatorType);
         ComparatorBase comp = (ComparatorBase) comparatorClass.newInstance();
-        compareOutput = comp.compare(CommandExecutor.getLastCommandOutput(), 
+        compareOutput = comp.compare(cmdResult.getCommandOutput(), 
           compdata.getExpectedOutput());
       } catch (Exception e) {
         LOG.info("Error in instantiating the comparator" + e);
@@ -351,9 +310,10 @@
    
       // Execute the test commands
       ArrayList<TestCmd> testCommands = testdata.getTestCommands();
+      Result cmdResult = null;
       for (TestCmd cmd : testCommands) {
       try {
-        CommandExecutor.executeCommand(cmd, namenode, jobtracker);
+        cmdResult = execute(cmd);
       } catch (Exception e) {
         fail(StringUtils.stringifyException(e));
       }
@@ -368,12 +328,12 @@
         boolean compareOutput = false;
         
         if (! comptype.equalsIgnoreCase("none")) {
-          compareOutput = compareTestOutput(cd);
+          compareOutput = compareTestOutput(cd, cmdResult);
           overallTCResult &= compareOutput;
         }
         
-        cd.setExitCode(CommandExecutor.getLastExitCode());
-        cd.setActualOutput(CommandExecutor.getLastCommandOutput());
+        cd.setExitCode(cmdResult.getExitCode());
+        cd.setActualOutput(cmdResult.getCommandOutput());
         cd.setTestResult(compareOutput);
       }
       testdata.setTestResult(overallTCResult);
@@ -382,7 +342,7 @@
       ArrayList<TestCmd> cleanupCommands = testdata.getCleanupCommands();
       for (TestCmd cmd : cleanupCommands) {
       try { 
-        CommandExecutor.executeCommand(cmd, namenode, jobtracker);
+        execute(cmd);
       } catch (Exception e) {
         fail(StringUtils.stringifyException(e));
       }
@@ -390,10 +350,14 @@
     }
   }
   
+  protected CommandExecutor.Result execute(TestCmd cmd) throws Exception {
+    throw new Exception("Unknow type of Test command:"+ cmd.getType());
+  }
+  
   /*
    * Parser class for the test config xml file
    */
-  static class TestConfigFileParser extends DefaultHandler {
+  class TestConfigFileParser extends DefaultHandler {
     String charString = null;
     CLITestData td = null;
     ArrayList<TestCmd> testCommands = null;

Added: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java?rev=771635&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java Tue May  5 09:16:51
2009
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CommandExecutor;
+import org.apache.hadoop.cli.util.CLITestData.TestCmd;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.util.ToolRunner;
+
+public class TestHDFSCLI extends TestCLI{
+
+  protected MiniDFSCluster dfsCluster = null;
+  protected DistributedFileSystem dfs = null;
+  protected String namenode = null;
+  protected DFSAdminCmdExecutor dfsAdmCmdExecutor = null;
+  protected FSCmdExecutor fsCmdExecutor = null;
+  
+  public void setUp() throws Exception {
+    super.setUp();
+    // Many of the tests expect a replication value of 1 in the output
+    conf.setInt("dfs.replication", 1);
+    
+    // Build racks and hosts configuration to test dfsAdmin -printTopology
+    String [] racks =  {"/rack1", "/rack1", "/rack2", "/rack2",
+                        "/rack2", "/rack3", "/rack4", "/rack4" };
+    String [] hosts = {"host1", "host2", "host3", "host4",
+                       "host5", "host6", "host7", "host8" };
+    dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
+    
+    namenode = conf.get("fs.default.name", "file:///");
+    
+    username = System.getProperty("user.name");
+    dfsAdmCmdExecutor = new DFSAdminCmdExecutor(namenode);
+    fsCmdExecutor =  new FSCmdExecutor(namenode);
+
+    FileSystem fs = dfsCluster.getFileSystem();
+    assertTrue("Not a HDFS: "+fs.getUri(),
+               fs instanceof DistributedFileSystem);
+    dfs = (DistributedFileSystem) fs;
+  }
+
+  protected String getTestFile() {
+    return "testHDFSConf.xml";
+  }
+  
+  public void tearDown() throws Exception {
+    dfs.close();
+    dfsCluster.shutdown();
+    Thread.sleep(2000);
+    super.tearDown();
+  }
+
+  protected String expandCommand(final String cmd) {
+    String expCmd = cmd;
+    expCmd = expCmd.replaceAll("NAMENODE", namenode);
+    expCmd = super.expandCommand(cmd);
+    return expCmd;
+  }
+  
+  protected Result execute(TestCmd cmd) throws Exception {
+    CommandExecutor executor = null;
+    switch(cmd.getType()) {
+    case DFSADMIN:
+      executor = dfsAdmCmdExecutor;
+      break;
+    case FS:
+      executor = fsCmdExecutor;
+      break;
+    default:
+      throw new Exception("Unknow type of Test command:"+ cmd.getType());
+    }
+    return executor.executeCommand(cmd.getCmd());
+  }
+  
+  public static class DFSAdminCmdExecutor extends CommandExecutor {
+    private String namenode = null;
+    public DFSAdminCmdExecutor(String namenode) {
+      this.namenode = namenode;
+    }
+    
+    protected void execute(final String cmd) throws Exception{
+      DFSAdmin shell = new DFSAdmin();
+      String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
+      ToolRunner.run(shell, args);
+    }
+  }
+  
+  public static class FSCmdExecutor extends CommandExecutor {
+    private String namenode = null;
+    public FSCmdExecutor(String namenode) {
+      this.namenode = namenode;
+    }
+    protected void execute(final String cmd) throws Exception{
+      FsShell shell = new FsShell();
+      String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
+      ToolRunner.run(shell, args);
+    }
+  }
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java?rev=771635&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java Tue May  5 09:16:51 2009
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CommandExecutor;
+import org.apache.hadoop.cli.util.CLITestData.TestCmd;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.tools.MRAdmin;
+import org.apache.hadoop.util.ToolRunner;
+
+public class TestMRCLI extends TestHDFSCLI{
+
+  protected MiniMRCluster mrCluster = null;
+  protected String jobtracker = null;
+  protected MRCmdExecutor cmdExecutor = null;
+  
+  public void setUp() throws Exception {
+    super.setUp();
+    JobConf mrConf = new JobConf(conf);
+    mrCluster = new MiniMRCluster(1, dfsCluster.getFileSystem().getUri().toString(), 1, 
+                           null, null, mrConf);
+    jobtracker = mrCluster.createJobConf().get("mapred.job.tracker", "local");
+    cmdExecutor = new MRCmdExecutor(jobtracker);
+  }
+
+  
+  public void tearDown() throws Exception {
+    mrCluster.shutdown();
+    super.tearDown();
+  }
+  
+  protected String getTestFile() {
+    return "testMRConf.xml";
+  }
+  
+  protected String expandCommand(final String cmd) {
+    String expCmd = cmd;
+    expCmd = expCmd.replaceAll("JOBTRACKER", jobtracker);
+    expCmd = super.expandCommand(cmd);
+    return expCmd;
+  }
+  
+  protected Result execute(TestCmd cmd) throws Exception {
+    if(cmd.getType() == TestCmd.CommandType.MRADMIN) {
+      return cmdExecutor.executeCommand(cmd.getCmd());
+    }
+    else throw new Exception("Unknow type of Test command:"+ cmd.getType());
+  }
+  
+  public static class MRCmdExecutor extends CommandExecutor {
+    private String jobtracker = null;
+    public MRCmdExecutor(String jobtracker) {
+      this.jobtracker = jobtracker;
+    }
+    protected void execute(final String cmd) throws Exception{
+      MRAdmin mradmin = new MRAdmin();
+      String[] args = getCommandAsArgs(cmd, "JOBTRACKER", jobtracker);
+      ToolRunner.run(mradmin, args);
+    }
+  }
+}



Mime
View raw message