hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1310547 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/
Date Fri, 06 Apr 2012 18:57:44 GMT
Author: bobby
Date: Fri Apr  6 18:57:44 2012
New Revision: 1310547

URL: http://svn.apache.org/viewvc?rev=1310547&view=rev
Log:
svn merge -c 1310546 FIXES: MAPREDUCE-4110. Fix tests in TestMiniMRClasspath & TestMiniMRWithDFSWithDistinctUsers
(Devaraj K via bobby)

Modified:
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Apr  6 18:57:44
2012
@@ -114,7 +114,11 @@ Release 2.0.0 - UNRELEASED
 
     MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
     K via bobby)
- 
+
+    MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
+    org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
+    bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
Fri Apr  6 18:57:44 2012
@@ -18,47 +18,43 @@
 
 package org.apache.hadoop.mapred;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.URI;
 
 import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.junit.Ignore;
+import org.junit.Assert;
+import org.junit.Test;
 
 /**
  * A JUnit test to test Mini Map-Reduce Cluster with multiple directories
  * and check for correct classpath
  */
-@Ignore
-public class TestMiniMRClasspath extends TestCase {
+public class TestMiniMRClasspath {
   
   
-  static void configureWordCount(FileSystem fs,
-                                String jobTracker,
-                                JobConf conf,
-                                String input,
-                                int numMaps,
-                                int numReduces,
-                                Path inDir, Path outDir) throws IOException {
+  static void configureWordCount(FileSystem fs, JobConf conf, String input,
+      int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
     fs.delete(outDir, true);
     if (!fs.mkdirs(inDir)) {
       throw new IOException("Mkdirs failed to create " + inDir.toString());
     }
-    {
-      DataOutputStream file = fs.create(new Path(inDir, "part-0"));
-      file.writeBytes(input);
-      file.close();
-    }
+    DataOutputStream file = fs.create(new Path(inDir, "part-0"));
+    file.writeBytes(input);
+    file.close();
     FileSystem.setDefaultUri(conf, fs.getUri());
-    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
-    conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
     conf.setJobName("wordcount");
     conf.setInputFormat(TextInputFormat.class);
     
@@ -74,18 +70,17 @@ public class TestMiniMRClasspath extends
     FileOutputFormat.setOutputPath(conf, outDir);
     conf.setNumMapTasks(numMaps);
     conf.setNumReduceTasks(numReduces);
-    //pass a job.jar already included in the hadoop build
-    conf.setJar("build/test/mapred/testjar/testjob.jar");
+    //set the tests jar file
+    conf.setJarByClass(TestMiniMRClasspath.class);
   }
   
-  static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
-                                String input, int numMaps, int numReduces) 
+  static String launchWordCount(URI fileSys, JobConf conf, String input,
+                                int numMaps, int numReduces) 
   throws IOException {
     final Path inDir = new Path("/testing/wc/input");
     final Path outDir = new Path("/testing/wc/output");
     FileSystem fs = FileSystem.get(fileSys, conf);
-    configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir, 
-                       outDir);
+    configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
     JobClient.runJob(conf);
     StringBuffer result = new StringBuffer();
     {
@@ -107,8 +102,8 @@ public class TestMiniMRClasspath extends
     return result.toString();
   }
 
-  static String launchExternal(URI uri, String jobTracker, JobConf conf,
-                               String input, int numMaps, int numReduces)
+  static String launchExternal(URI uri, JobConf conf, String input,
+                               int numMaps, int numReduces)
     throws IOException {
 
     final Path inDir = new Path("/testing/ext/input");
@@ -124,8 +119,7 @@ public class TestMiniMRClasspath extends
       file.close();
     }
     FileSystem.setDefaultUri(conf, uri);
-    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
-    conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+    conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
     conf.setJobName("wordcount");
     conf.setInputFormat(TextInputFormat.class);
 
@@ -142,8 +136,8 @@ public class TestMiniMRClasspath extends
     conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer"); 
     conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
 
-    //pass a job.jar already included in the hadoop build
-    conf.setJar("build/test/mapred/testjar/testjob.jar");
+    // set the tests jar file
+    conf.setJarByClass(TestMiniMRClasspath.class);
     JobClient.runJob(conf);
     StringBuffer result = new StringBuffer();
     Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
@@ -164,6 +158,7 @@ public class TestMiniMRClasspath extends
     return result.toString();
   }
    
+  @Test
   public void testClassPath() throws IOException {
     String namenode = null;
     MiniDFSCluster dfs = null;
@@ -180,13 +175,10 @@ public class TestMiniMRClasspath extends
       mr = new MiniMRCluster(taskTrackers, namenode, 3);
       JobConf jobConf = new JobConf();
       String result;
-      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
-      result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf, 
-                               "The quick brown fox\nhas many silly\n" + 
-                               "red fox sox\n",
-                               3, 1);
-      assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
-                   "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
+      result = launchWordCount(fileSys.getUri(), jobConf,
+          "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
+      Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
+          + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
           
     } finally {
       if (dfs != null) { dfs.shutdown(); }
@@ -195,6 +187,7 @@ public class TestMiniMRClasspath extends
     }
   }
   
+  @Test
   public void testExternalWritable()
     throws IOException {
  
@@ -214,12 +207,10 @@ public class TestMiniMRClasspath extends
       mr = new MiniMRCluster(taskTrackers, namenode, 3);      
       JobConf jobConf = new JobConf();
       String result;
-      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
       
-      result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf, 
-                              "Dennis was here!\nDennis again!",
-                              3, 1);
-      assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
+      result = launchExternal(fileSys.getUri(), jobConf,
+          "Dennis was here!\nDennis again!", 3, 1);
+      Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
       
     } 
     finally {

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
Fri Apr  6 18:57:44 2012
@@ -17,26 +17,26 @@
  */
 package org.apache.hadoop.mapred;
 
-import java.io.*;
+import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.security.*;
-import org.junit.Ignore;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
  */
-@Ignore
-public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
+public class TestMiniMRWithDFSWithDistinctUsers {
   static final UserGroupInformation DFS_UGI = createUGI("dfs", true); 
   static final UserGroupInformation ALICE_UGI = createUGI("alice", false); 
   static final UserGroupInformation BOB_UGI = createUGI("bob", false); 
@@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistin
   MiniDFSCluster dfs = null;
   FileSystem fs = null;
   Configuration conf = new Configuration();
-  String jobTrackerName;
 
   static UserGroupInformation createUGI(String name, boolean issuper) {
     String group = issuper? "supergroup": name;
@@ -71,9 +70,10 @@ public class TestMiniMRWithDFSWithDistin
       });
 
     rj.waitForCompletion();
-    assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
+    Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
   }
 
+  @Before
   public void setUp() throws Exception {
     dfs = new MiniDFSCluster(conf, 4, true, null);
 
@@ -96,29 +96,30 @@ public class TestMiniMRWithDFSWithDistin
 
     mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
                            1, null, null, MR_UGI, mrConf);
-    jobTrackerName = "localhost:" + mr.getJobTrackerPort();
   }
 
+  @After
   public void tearDown() throws Exception {
     if (mr != null) { mr.shutdown();}
     if (dfs != null) { dfs.shutdown(); }
   }
   
+  @Test
   public void testDistinctUsers() throws Exception {
     JobConf job1 = mr.createJobConf();
     String input = "The quick brown fox\nhas many silly\n" 
       + "red fox sox\n";
     Path inDir = new Path("/testing/distinct/input");
     Path outDir = new Path("/user/alice/output");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
-                                           input, 2, 1, inDir, outDir);
+    TestMiniMRClasspath
+        .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
     runJobAsUser(job1, ALICE_UGI);
 
     JobConf job2 = mr.createJobConf();
     Path inDir2 = new Path("/testing/distinct/input2");
     Path outDir2 = new Path("/user/bob/output2");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2, 
-                                           input, 2, 1, inDir2, outDir2);
+    TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
+        outDir2);
     runJobAsUser(job2, BOB_UGI);
   }
 
@@ -127,6 +128,7 @@ public class TestMiniMRWithDFSWithDistin
    * task makes lots of spills (more than fit in the spill index cache)
    * that it will succeed.
    */
+  @Test
   public void testMultipleSpills() throws Exception {
     JobConf job1 = mr.createJobConf();
 
@@ -141,8 +143,8 @@ public class TestMiniMRWithDFSWithDistin
       + "red fox sox\n";
     Path inDir = new Path("/testing/distinct/input");
     Path outDir = new Path("/user/alice/output");
-    TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, 
-                                           input, 2, 1, inDir, outDir);
+    TestMiniMRClasspath
+        .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
     runJobAsUser(job1, ALICE_UGI);
   }
 }



Mime
View raw message