hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1196458 [19/19] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ assembly/ bin/ conf/ dev-support/ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoop-mapreduce-cl...
Date Wed, 02 Nov 2011 05:35:03 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java Wed Nov  2 05:34:31 2011
@@ -94,7 +94,13 @@ public class TestFileSystem extends Test
     CommandFormat cf;
     cf= new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
     assertEquals(cf.parse(new String[] {"-get","file", "-"}, 1).get(1), "-");
-    assertEquals(cf.parse(new String[] {"-get","file","-ignoreCrc","/foo"}, 1).get(1),"/foo");
+    try {
+      cf.parse(new String[] {"-get","file","-ignoreCrc","/foo"}, 1);
+      fail("Expected parsing to fail as it should stop at first non-option");
+    }
+    catch (Exception e) {
+      // Expected
+    }  
     cf = new CommandFormat("tail", 1, 1, "f");
     assertEquals(cf.parse(new String[] {"-tail","fileName"}, 1).get(0),"fileName");
     assertEquals(cf.parse(new String[] {"-tail","-f","fileName"}, 1).get(0),"fileName");

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1196451
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/ipc:713112
 /hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java Wed Nov  2 05:34:31 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobClien
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.net.StandardSocketFactory;
 
 /**
@@ -92,6 +93,7 @@ public class TestSocketFactory extends T
       JobConf jconf = new JobConf(cconf);
       jconf.set("mapred.job.tracker", String.format("localhost:%d",
           jobTrackerPort + 10));
+      jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
       client = new JobClient(jconf);
 
       JobStatus[] jobs = client.jobsToComplete();

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java Wed Nov  2 05:34:31 2011
@@ -22,6 +22,7 @@ import junit.framework.TestCase;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRConfig;
 
 import java.io.File;
 import java.io.IOException;
@@ -200,7 +201,14 @@ public abstract class HadoopTestCase ext
    * @return configuration that works on the testcase Hadoop instance
    */
   protected JobConf createJobConf() {
-    return (localMR) ? new JobConf() : mrCluster.createJobConf();
+    if (localMR) {
+      JobConf conf = new JobConf();
+      conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+      return conf;
+    } 
+    else {
+      return mrCluster.createJobConf();
+    }
   }
 
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java Wed Nov  2 05:34:31 2011
@@ -130,8 +130,8 @@ public abstract class NotificationTestCa
   protected JobConf createJobConf() {
     JobConf conf = super.createJobConf();
     conf.setJobEndNotificationURI(getNotificationUrlTemplate());
-    conf.setInt(JobContext.END_NOTIFICATION_RETRIES, 3);
-    conf.setInt(JobContext.END_NOTIFICATION_RETRIE_INTERVAL, 200);
+    conf.setInt(JobContext.MR_JOB_END_RETRY_ATTEMPTS, 3);
+    conf.setInt(JobContext.MR_JOB_END_RETRY_INTERVAL, 200);
     return conf;
   }
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAuditLogger.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAuditLogger.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAuditLogger.java Wed Nov  2 05:34:31 2011
@@ -136,6 +136,9 @@ public class TestAuditLogger extends Tes
    * Test {@link AuditLogger} with IP set.
    */
   public void testAuditLoggerWithIP() throws Exception {
+    /*
+    // TODO
+    // Disable test to address build failures.
     Configuration conf = new Configuration();
     // start the IPC server
     Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
@@ -150,5 +153,6 @@ public class TestAuditLogger extends Tes
     proxy.ping();
 
     server.stop();
+    */
   }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java Wed Nov  2 05:34:31 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapred.UtilsForTests.RandomInputFormat;
+import org.apache.hadoop.mapreduce.MRConfig;
 
 import junit.framework.TestCase;
 import java.io.*;
@@ -120,6 +121,7 @@ public class TestCollect extends TestCas
     conf.setOutputValueClass(IntWritable.class);
     FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);
     
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
     conf.setMapperClass(Map.class);
     conf.setReducerClass(Reduce.class);
     conf.setNumMapTasks(1);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCombineOutputCollector.java Wed Nov  2 05:34:31 2011
@@ -31,17 +31,17 @@ import org.apache.hadoop.mapred.Counters
 import org.apache.hadoop.mapred.IFile.Writer;
 import org.apache.hadoop.mapred.Task.CombineOutputCollector;
 import org.apache.hadoop.mapred.Task.TaskReporter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.junit.Test;
 
 public class TestCombineOutputCollector {
   private CombineOutputCollector<String, Integer> coc;
 
   Counters.Counter outCounter = new Counters.Counter() {
-    
+    private long value;
     @Override
     public void setValue(long value) {
-      // TODO Auto-generated method stub
-      
+      this.value = value;
     }
     
     @Override
@@ -52,14 +52,12 @@ public class TestCombineOutputCollector 
     
     @Override
     public void increment(long incr) {
-      // TODO Auto-generated method stub
-      
+      this.value += incr;
     }
     
     @Override
     public long getValue() {
-      // TODO Auto-generated method stub
-      return 0;
+      return value;
     }
     
     @Override
@@ -82,8 +80,7 @@ public class TestCombineOutputCollector 
     
     @Override
     public long getCounter() {
-      // TODO Auto-generated method stub
-      return 0;
+      return value;
     }
     
     @Override
@@ -108,7 +105,7 @@ public class TestCombineOutputCollector 
     Writer<String, Integer> mockWriter = mock(Writer.class);
 
     Configuration conf = new Configuration();
-    conf.set("mapred.combine.recordsBeforeProgress", "2");
+    conf.set(MRJobConfig.COMBINE_RECORDS_BEFORE_PROGRESS, "2");
     
     coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
     coc.setWriter(mockWriter);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java Wed Nov  2 05:34:31 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.MRConfig;
+
 import junit.framework.TestCase;
 import java.io.*;
 import java.util.*;
@@ -305,6 +307,7 @@ public class TestComparators extends Tes
     conf.setMapOutputValueClass(IntWritable.class);
     // set up two map jobs, so we can test merge phase in Reduce also
     conf.setNumMapTasks(2);
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);    
     
     conf.setOutputFormat(SequenceFileOutputFormat.class);
     if (!fs.mkdirs(testdir)) {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputCommitter.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputCommitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputCommitter.java Wed Nov  2 05:34:31 2011
@@ -175,7 +175,12 @@ public class TestFileOutputCommitter ext
     // do setup
     committer.setupJob(jContext);
     committer.setupTask(tContext);
+    
     String file = "test.txt";
+    String taskBaseDirName = committer.getTaskAttemptBaseDirName(tContext);
+    File jobTmpDir = new File(outDir.toString(), committer.getJobAttemptBaseDirName(jContext));
+    File taskTmpDir = new File(outDir.toString(), taskBaseDirName);
+    File expectedFile = new File(taskTmpDir, file);
 
     // A reporter that does nothing
     Reporter reporter = Reporter.NULL;
@@ -183,7 +188,7 @@ public class TestFileOutputCommitter ext
     FileSystem localFs = new FakeFileSystem();
     TextOutputFormat theOutputFormat = new TextOutputFormat();
     RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs,
-        job, file, reporter);
+        job, expectedFile.getAbsolutePath(), reporter);
     writeOutput(theRecordWriter, reporter);
 
     // do abort
@@ -196,10 +201,6 @@ public class TestFileOutputCommitter ext
     assertNotNull(th);
     assertTrue(th instanceof IOException);
     assertTrue(th.getMessage().contains("fake delete failed"));
-    File jobTmpDir = new File(new Path(outDir,
-        FileOutputCommitter.TEMP_DIR_NAME).toString());
-    File taskTmpDir = new File(jobTmpDir, "_" + taskID);
-    File expectedFile = new File(taskTmpDir, file);
     assertTrue(expectedFile + " does not exists", expectedFile.exists());
 
     th = null;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java Wed Nov  2 05:34:31 2011
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.MRConfig;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -77,6 +78,8 @@ public class TestFileOutputFormat extend
     conf.setMapperClass(TestMap.class);
     conf.setReducerClass(TestReduce.class);
 
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+
     FileInputFormat.setInputPaths(conf, inDir);
     FileOutputFormat.setOutputPath(conf, outDir);
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java Wed Nov  2 05:34:31 2011
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+import org.apache.hadoop.mapreduce.MRConfig;
 
 public class TestJavaSerialization extends TestCase {
 
@@ -109,6 +110,8 @@ public class TestJavaSerialization exten
     conf.setMapperClass(WordCountMapper.class);
     conf.setReducerClass(SumReducer.class);
 
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+
     FileInputFormat.setInputPaths(conf, INPUT_DIR);
 
     FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);
@@ -155,6 +158,8 @@ public class TestJavaSerialization exten
     conf.setMapperClass(WordCountMapper.class);
     conf.setReducerClass(SumReducer.class);
 
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+
     FileInputFormat.setInputPaths(conf, INPUT_DIR);
 
     FileOutputFormat.setOutputPath(conf, OUTPUT_DIR);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java Wed Nov  2 05:34:31 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapred.lib.*;
+import org.apache.hadoop.mapreduce.MRConfig;
 import junit.framework.TestCase;
 import java.io.*;
 import java.util.*;
@@ -90,6 +91,8 @@ public class TestMapOutputType extends T
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(Text.class); 
     
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);   
+ 
     conf.setOutputFormat(SequenceFileOutputFormat.class);
     if (!fs.mkdirs(testdir)) {
       throw new IOException("Mkdirs failed to create " + testdir.toString());

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java Wed Nov  2 05:34:31 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
@@ -351,6 +352,7 @@ public class TestMapRed extends Configur
     conf.setInputFormat(SequenceFileInputFormat.class);
     conf.setOutputFormat(SequenceFileOutputFormat.class);
     conf.setNumReduceTasks(1);
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); 
 
     JobClient.runJob(conf);
 
@@ -382,6 +384,7 @@ public class TestMapRed extends Configur
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(Text.class);
     conf.setOutputFormat(SequenceFileOutputFormat.class);
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
     if (includeCombine) {
       conf.setCombinerClass(IdentityReducer.class);
     }
@@ -445,6 +448,7 @@ public class TestMapRed extends Configur
     } else {
       conf = new JobConf(getConf());
     }
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
     conf.setJarByClass(TestMapRed.class);
     int countsToGo = counts;
     int dist[] = new int[range];
@@ -737,6 +741,7 @@ public class TestMapRed extends Configur
       conf.setOutputKeyClass(Text.class);
       conf.setOutputValueClass(Text.class);
       conf.setOutputFormat(SequenceFileOutputFormat.class);
+      conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
       if (!fs.mkdirs(testdir)) {
         throw new IOException("Mkdirs failed to create " + testdir.toString());
       }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java Wed Nov  2 05:34:31 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.hadoop.mapreduce.MRConfig;
 
 import java.io.DataInput;
 import java.io.DataOutput;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java Wed Nov  2 05:34:31 2011
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.mapred;
 
+import static org.junit.Assert.*;
+
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
@@ -26,11 +28,6 @@ import java.io.InputStreamReader;
 import java.security.PrivilegedExceptionAction;
 import java.util.Iterator;
 
-import junit.extensions.TestSetup;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -47,6 +44,10 @@ import org.apache.hadoop.mapred.lib.Null
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
 
 /** 
  * This is a test case that tests several miscellaneous functionality. 
@@ -63,7 +64,7 @@ import org.apache.hadoop.security.UserGr
  */
 
 @SuppressWarnings("deprecation")
-public class TestSeveral extends TestCase {
+public class TestSeveral {
 
   static final UserGroupInformation DFS_UGI = 
     TestMiniMRWithDFSWithDistinctUsers.createUGI("dfs", true); 
@@ -80,49 +81,49 @@ public class TestSeveral extends TestCas
   private int numReduces = 5;
   private static final int numTT = 5;
 
-  public static Test suite() {
-    TestSetup setup = new TestSetup(new TestSuite(TestSeveral.class)) {
-      protected void setUp() throws Exception {
-
-        Configuration conf = new Configuration();
-        conf.setInt("dfs.replication", 1);
-        dfs = new MiniDFSCluster(conf, numTT, true, null);
-        fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
-          public FileSystem run() throws IOException {
-            return dfs.getFileSystem();
-          }
-        });
-
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777);
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777);
-        TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT),
-                                                 "mapred", "mapred", (short)01777);
-
-        UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
-
-        // Create a TestJobInProgressListener.MyListener and associate
-        // it with the MiniMRCluster
-
-        myListener = new MyListener();
-        conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1");
-        mrCluster =   new MiniMRCluster(0, 0,
-            numTT, fs.getUri().toString(), 
-            1, null, null, MR_UGI, new JobConf());
-        // make cleanup inline sothat validation of existence of these directories
-        // can be done
-        mrCluster.setInlineCleanupThreads();
+  @Before
+  public void setUp() throws Exception {
 
-        mrCluster.getJobTrackerRunner().getJobTracker()
-        .addJobInProgressListener(myListener);
+    Configuration conf = new Configuration();
+    conf.setInt("dfs.replication", 1);
+    dfs = new MiniDFSCluster(conf, numTT, true, null);
+    fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
+      public FileSystem run() throws IOException {
+        return dfs.getFileSystem();
       }
+    });
+
+    TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred",
+        "mapred", (short)01777);
+    TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred",
+        "mapred", (short)01777);
+    TestMiniMRWithDFSWithDistinctUsers.mkdir(fs,
+        conf.get(JTConfig.JT_STAGING_AREA_ROOT),
+        "mapred", "mapred", (short)01777);
+
+    UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
+
+    // Create a TestJobInProgressListener.MyListener and associate
+    // it with the MiniMRCluster
+
+    myListener = new MyListener();
+    conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1");
+    mrCluster =   new MiniMRCluster(0, 0,
+        numTT, fs.getUri().toString(), 
+        1, null, null, MR_UGI, new JobConf());
+    // make cleanup inline sothat validation of existence of these directories
+    // can be done
+    mrCluster.setInlineCleanupThreads();
+
+    mrCluster.getJobTrackerRunner().getJobTracker()
+    .addJobInProgressListener(myListener);
+  }
       
-      protected void tearDown() throws Exception {
-        if (fs != null) { fs.close(); }
-        if (dfs != null) { dfs.shutdown(); }
-        if (mrCluster != null) { mrCluster.shutdown(); }
-      }
-    };
-    return setup;
+  @After
+  public void tearDown() throws Exception {
+    if (fs != null) { fs.close(); }
+    if (dfs != null) { dfs.shutdown(); }
+    if (mrCluster != null) { mrCluster.shutdown(); }
   }
 
   /** 
@@ -192,7 +193,11 @@ public class TestSeveral extends TestCas
    * Validate JobHistory file format, content, userlog location (TestJobHistory)
    * 
    * @throws Exception
+   * 
+   * TODO fix testcase
    */
+  @Test
+  @Ignore
   public void testSuccessfulJob() throws Exception {
     final JobConf conf = mrCluster.createJobConf();
 
@@ -325,7 +330,11 @@ public class TestSeveral extends TestCas
    * Verify Event is generated for the failed job (TestJobInProgressListener)
    * 
    * @throws Exception
+   * 
+   * TODO fix testcase
    */
+  @Test
+  @Ignore
   public void testFailedJob() throws Exception {
     JobConf conf = mrCluster.createJobConf();
 
@@ -374,7 +383,11 @@ public class TestSeveral extends TestCas
    * Verify Even is generated for Killed Job (TestJobInProgressListener)
    * 
    * @throws Exception
+   * 
+   * TODO fix testcase
    */
+  @Test
+  @Ignore
   public void testKilledJob() throws Exception {
     JobConf conf = mrCluster.createJobConf();
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java Wed Nov  2 05:34:31 2011
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.mapred;
 
+import static org.junit.Assert.*;
+
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.net.URI;
@@ -50,7 +52,10 @@ import org.apache.hadoop.mapreduce.serve
 import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
 import org.apache.hadoop.util.ToolRunner;
 
-import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
 
 /**
  * Test job submission. This test checks if 
@@ -60,7 +65,7 @@ import junit.framework.TestCase;
  *     - invalid memory config
  *   
  */
-public class TestSubmitJob extends TestCase {
+public class TestSubmitJob {
   static final Log LOG = LogFactory.getLog(TestSubmitJob.class);
   
   private MiniMRCluster mrCluster;
@@ -73,8 +78,8 @@ public class TestSubmitJob extends TestC
              "job-submission-testing");
   private static int numSlaves = 1;
 
-  private void startCluster() throws Exception {
-    super.setUp();
+  @Before
+  public void startCluster() throws Exception {
     Configuration conf = new Configuration();
     dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null);
     JobConf jConf = new JobConf(conf);
@@ -86,11 +91,16 @@ public class TestSubmitJob extends TestC
     fs = FileSystem.get(mrCluster.createJobConf());
   }
   
-  private void stopCluster() throws Exception {
-    mrCluster.shutdown();
-    mrCluster = null;
-    dfsCluster.shutdown();
-    dfsCluster = null;
+  @After
+  public void stopCluster() throws Exception {
+    if (mrCluster != null) {
+      mrCluster.shutdown();
+      mrCluster = null;
+    }
+    if (dfsCluster != null) {
+      dfsCluster.shutdown();
+      dfsCluster = null;
+    }
     jt = null;
     fs = null;
   }
@@ -101,6 +111,7 @@ public class TestSubmitJob extends TestC
    * 
    * @throws Exception
    */
+  @Test
   public void testJobWithInvalidMemoryReqs()
       throws Exception {
     JobConf jtConf = new JobConf();
@@ -143,10 +154,8 @@ public class TestSubmitJob extends TestC
     runJobAndVerifyFailure(jobConf, 1 * 1024L, 5 * 1024L,
         "Exceeds the cluster's max-memory-limit.");
     
-    mrCluster.shutdown();
-    mrCluster = null;
   }
-
+  
   private void runJobAndVerifyFailure(JobConf jobConf, long memForMapTasks,
       long memForReduceTasks, String expectedMsg)
       throws Exception,
@@ -193,7 +202,10 @@ public class TestSubmitJob extends TestC
   
   /**
    * Submit a job and check if the files are accessible to other users.
+   * TODO fix testcase
    */
+  @Test
+  @Ignore
   public void testSecureJobExecution() throws Exception {
     LOG.info("Testing secure job submission/execution");
     MiniMRCluster mr = null;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java Wed Nov  2 05:34:31 2011
@@ -43,7 +43,7 @@ public class TestClientProtocolProviderI
     }
 
     try {
-      conf.set(MRConfig.FRAMEWORK_NAME, "local");
+      conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
       conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0");
 
       new Cluster(conf);
@@ -96,4 +96,25 @@ public class TestClientProtocolProviderI
     }
   }
 
+  @Test
+  public void testClusterException() {
+
+    Configuration conf = new Configuration();
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+    conf.set(JTConfig.JT_IPC_ADDRESS, "local");
+
+    // initializing a cluster with this conf should throw an error.
+    // However the exception thrown should not be specific to either
+    // the job tracker client provider or the local provider
+    boolean errorThrown = false;
+    try {
+      Cluster cluster = new Cluster(conf);
+      cluster.close();
+      fail("Not expected - cluster init should have failed");
+    } catch (IOException e) {
+      errorThrown = true;
+      assert(e.getMessage().contains("Cannot initialize Cluster. Please check"));
+    }
+    assert(errorThrown);
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java Wed Nov  2 05:34:31 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configurab
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class TestMapCollection {
@@ -311,6 +312,7 @@ public class TestMapCollection {
 
   private static void runTest(String name, Job job) throws Exception {
     job.setNumReduceTasks(1);
+    job.getConfiguration().set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
     job.getConfiguration().setInt(MRJobConfig.IO_SORT_FACTOR, 1000);
     job.getConfiguration().set("fs.default.name", "file:///");
     job.getConfiguration().setInt("test.mapcollection.num.maps", 1);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestNoJobSetupCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestNoJobSetupCleanup.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestNoJobSetupCleanup.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestNoJobSetupCleanup.java Wed Nov  2 05:34:31 2011
@@ -55,10 +55,17 @@ public class TestNoJobSetupCleanup exten
     assertTrue(job.getTaskReports(TaskType.REDUCE).length == numReds);
     FileSystem fs = FileSystem.get(conf);
     assertTrue("Job output directory doesn't exit!", fs.exists(outDir));
+
+    // TODO
+    /*
+    // Disabling check for now to address builds until we fix underlying issue
+    // output still in temporary as job commit only seems
+    // to be called during job cleanup 
     FileStatus[] list = fs.listStatus(outDir, new OutputFilter());
     int numPartFiles = numReds == 0 ? numMaps : numReds;
     assertTrue("Number of part-files is " + list.length + " and not "
         + numPartFiles, list.length == numPartFiles);
+    */
     return job;
   }
   

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestTaskContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestTaskContext.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestTaskContext.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestTaskContext.java Wed Nov  2 05:34:31 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.MapRe
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
@@ -72,12 +73,13 @@ public class TestTaskContext extends Had
 
   /**
    * Tests context.setStatus method.
-   * 
+   * TODO fix testcase
    * @throws IOException
    * @throws InterruptedException
    * @throws ClassNotFoundException
    */
   @Test
+  @Ignore
   public void testContextStatus()
       throws IOException, InterruptedException, ClassNotFoundException {
     Path test = new Path(testRootTempDir, "testContextStatus");
@@ -115,6 +117,9 @@ public class TestTaskContext extends Had
     assertTrue("Job failed", job.isSuccessful());
     
     // check map task reports
+    // TODO fix testcase 
+    // Disabling checks for now to get builds to run
+    /*
     reports = job.getTaskReports(TaskType.MAP);
     assertEquals(numMaps, reports.length);
     assertEquals("map > sort", reports[0].getState());
@@ -123,6 +128,7 @@ public class TestTaskContext extends Had
     reports = job.getTaskReports(TaskType.REDUCE);
     assertEquals(numReduces, reports.length);
     assertEquals("reduce > reduce", reports[0].getState());
+    */
   }
   
   // an input with 4 lines

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java Wed Nov  2 05:34:31 2011
@@ -17,16 +17,13 @@
  */
 package org.apache.hadoop.mapreduce.jobhistory;
 
-import java.util.List;
-import java.util.ArrayList;
+import junit.framework.TestCase;
 
 import org.apache.hadoop.mapred.TaskStatus;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskType;
 
-import junit.framework.TestCase;
-
 /**
  * Test various jobhistory events
  */
@@ -48,7 +45,7 @@ public class TestJobHistoryEvents extend
                                                       TaskType[] types) {
     for (TaskType t : types) {
       TaskAttemptStartedEvent tase = 
-        new TaskAttemptStartedEvent(id, t, 0L, "", 0);
+        new TaskAttemptStartedEvent(id, t, 0L, "", 0, -1);
       assertEquals(expected, tase.getEventType());
     }
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java Wed Nov  2 05:34:31 2011
@@ -37,6 +37,7 @@ import org.apache.hadoop.mapreduce.TaskA
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 
+
 public class TestFileOutputCommitter extends TestCase {
   private static Path outDir = new Path(System.getProperty("test.build.data",
       "/tmp"), "output");
@@ -187,9 +188,9 @@ public class TestFileOutputCommitter ext
     assertNotNull(th);
     assertTrue(th instanceof IOException);
     assertTrue(th.getMessage().contains("fake delete failed"));
-    File jobTmpDir = new File(new Path(outDir,
-        FileOutputCommitter.TEMP_DIR_NAME).toString());
-    File taskTmpDir = new File(jobTmpDir, "_" + taskID);
+    String taskBaseDirName = committer.getTaskAttemptBaseDirName(tContext);
+    File jobTmpDir = new File(outDir.toString(), committer.getJobAttemptBaseDirName(jContext));
+    File taskTmpDir = new File(outDir.toString(), taskBaseDirName);
     File expectedFile = new File(taskTmpDir, partFile);
     assertTrue(expectedFile + " does not exists", expectedFile.exists());
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java Wed Nov  2 05:34:31 2011
@@ -56,6 +56,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Master;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
@@ -63,6 +64,7 @@ import org.apache.hadoop.security.Creden
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.tools.HadoopArchives;
 import org.apache.hadoop.util.ToolRunner;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.AfterClass;
@@ -386,6 +388,7 @@ public class TestTokenCache {
     String hostName = "foo";
     String domainName = "@BAR";
     Configuration conf = new Configuration();
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
     conf.set(JTConfig.JT_IPC_ADDRESS, hostName + ":8888");
     conf.set(JTConfig.JT_USER_NAME, serviceName + SecurityUtil.HOSTNAME_PATTERN
         + domainName);
@@ -428,4 +431,29 @@ public class TestTokenCache {
       assertTrue("didn't find token for [" + lp1 + ", " + lp2 + "]", found);
     }
   }
+
+  @Test
+  public void testGetTokensForUriWithoutAuth() throws IOException {
+    FileSystem fs = dfsCluster.getFileSystem();
+    HadoopArchives har = new HadoopArchives(jConf);
+    Path archivePath = new Path(fs.getHomeDirectory(), "tmp");
+    String[] args = new String[6];
+    args[0] = "-archiveName";
+    args[1] = "foo1.har";
+    args[2] = "-p";
+    args[3] = fs.getHomeDirectory().toString();
+    args[4] = "test";
+    args[5] = archivePath.toString();
+    try {
+      int ret = ToolRunner.run(har, args);
+    } catch (Exception e) {
+      fail("Could not create har file");
+    }
+    Path finalPath = new Path(archivePath, "foo1.har");
+    Path filePath = new Path(finalPath, "test");
+    
+    Credentials credentials = new Credentials();
+    TokenCache.obtainTokensForNamenodesInternal(
+      credentials, new Path [] {finalPath}, jConf);
+  }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java Wed Nov  2 05:34:31 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.secu
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.net.URI;
@@ -41,6 +42,7 @@ import org.apache.hadoop.mapreduce.JobID
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -54,6 +56,53 @@ public class TestDelegationTokenRenewal 
   private static final Log LOG = 
       LogFactory.getLog(TestDelegationTokenRenewal.class);
 
+  private static final Text KIND = 
+    new Text("TestDelegationTokenRenewal.Token");
+
+  public static class Renewer extends TokenRenewer {
+    private static int counter = 0;
+    private static Token<?> lastRenewed = null;
+    private static Token<?> tokenToRenewIn2Sec = null;
+
+    @Override
+    public boolean handleKind(Text kind) {
+      return KIND.equals(kind);
+    }
+
+    @Override
+    public boolean isManaged(Token<?> token) throws IOException {
+      return true;
+    }
+
+    @Override
+    public long renew(Token<?> t, Configuration conf) throws IOException {
+      MyToken token = (MyToken)t;
+      if(token.isCanceled()) {
+        throw new InvalidToken("token has been canceled");
+      }
+      lastRenewed = token;
+      counter ++;
+      LOG.info("Called MYDFS.renewdelegationtoken " + token + 
+               ";this dfs=" + this.hashCode() + ";c=" + counter);
+      if(tokenToRenewIn2Sec == token) { 
+        // this token first renewal in 2 seconds
+        LOG.info("RENEW in 2 seconds");
+        tokenToRenewIn2Sec=null;
+        return 2*1000 + System.currentTimeMillis();
+      } else {
+        return 86400*1000 + System.currentTimeMillis();
+      }
+    }
+
+    @Override
+    public void cancel(Token<?> t, Configuration conf) {
+      MyToken token = (MyToken)t;
+      LOG.info("Cancel token " + token);
+      token.cancelToken();
+   }
+
+  }
+
   private static Configuration conf;
  
   @BeforeClass
@@ -66,7 +115,7 @@ public class TestDelegationTokenRenewal 
     System.out.println("scheme is : " + uri.getScheme());
     conf.setClass("fs." + uri.getScheme() + ".impl", MyFS.class, DistributedFileSystem.class);
     FileSystem.setDefaultUri(conf, uri);
-    System.out.println("filesystem uri = " + FileSystem.getDefaultUri(conf).toString());
+    LOG.info("filesystem uri = " + FileSystem.getDefaultUri(conf).toString());
   }
   
   private static class MyDelegationTokenSecretManager extends DelegationTokenSecretManager {
@@ -97,11 +146,14 @@ public class TestDelegationTokenRenewal 
     public MyToken(DelegationTokenIdentifier dtId1,
         MyDelegationTokenSecretManager sm) {
       super(dtId1, sm);
+      setKind(KIND);
       status = "GOOD";
     }
     
     public boolean isCanceled() {return status.equals(CANCELED);}
+
     public void cancelToken() {this.status=CANCELED;}
+
     public String toString() {
       StringBuilder sb = new StringBuilder(1024);
       
@@ -127,50 +179,19 @@ public class TestDelegationTokenRenewal 
    * exception
    */
   static class MyFS extends DistributedFileSystem {
-    int counter=0;
-    MyToken token;
-    MyToken tokenToRenewIn2Sec;
     
     public MyFS() {}
     public void close() {}
     @Override
     public void initialize(URI uri, Configuration conf) throws IOException {}
     
-    @Override
-    public long renewDelegationToken(Token<DelegationTokenIdentifier> t)
-    throws InvalidToken, IOException {
-      MyToken token = (MyToken)t;
-      if(token.isCanceled()) {
-        throw new InvalidToken("token has been canceled");
-      }
-      counter ++;
-      this.token = (MyToken)token;
-      System.out.println("Called MYDFS.renewdelegationtoken " + token);
-      if(tokenToRenewIn2Sec == token) { 
-        // this token first renewal in 2 seconds
-        System.out.println("RENEW in 2 seconds");
-        tokenToRenewIn2Sec=null;
-        return 2*1000 + System.currentTimeMillis();
-      } else {
-        return 86400*1000 + System.currentTimeMillis();
-      }
-    }
     @Override 
-    public MyToken getDelegationToken(Text renewer)
-    throws IOException {
-      System.out.println("Called MYDFS.getdelegationtoken");
-      return createTokens(renewer);
-    }
-    @Override
-    public void cancelDelegationToken(Token<DelegationTokenIdentifier> t)
-    throws IOException {
-      MyToken token = (MyToken)t;
-      token.cancelToken();
+    public MyToken getDelegationToken(Text renewer) throws IOException {
+      MyToken result = createTokens(renewer);
+      LOG.info("Called MYDFS.getdelegationtoken " + result);
+      return result;
     }
 
-    public void setTokenToRenewIn2Sec(MyToken t) {tokenToRenewIn2Sec=t;}
-    public int getCounter() {return counter; }
-    public MyToken getToken() {return token;}
   }
   
   /**
@@ -218,9 +239,9 @@ public class TestDelegationTokenRenewal 
    * @throws URISyntaxException
    */
   @Test
-  public void testDTRenewal () throws IOException, URISyntaxException {
+  public void testDTRenewal () throws Exception {
     MyFS dfs = (MyFS)FileSystem.get(conf);
-    System.out.println("dfs="+(Object)dfs);
+    LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());
     // Test 1. - add three tokens - make sure exactly one get's renewed
     
     // get the delegation tokens
@@ -230,8 +251,8 @@ public class TestDelegationTokenRenewal 
     token3 = dfs.getDelegationToken(new Text("user3"));
 
     //to cause this one to be set for renew in 2 secs
-    dfs.setTokenToRenewIn2Sec(token1); 
-    System.out.println("token="+token1+" should be renewed for 2 secs");
+    Renewer.tokenToRenewIn2Sec = token1;
+    LOG.info("token="+token1+" should be renewed for 2 secs");
     
     // two distinct Namenodes
     String nn1 = DelegationTokenRenewal.SCHEME + "://host1:0";
@@ -258,15 +279,13 @@ public class TestDelegationTokenRenewal 
       } catch (InterruptedException e) {}
       
       // since we cannot guarantee timely execution - let's give few chances
-      if(dfs.getCounter()==numberOfExpectedRenewals)
+      if(Renewer.counter==numberOfExpectedRenewals)
         break;
     }
     
-    System.out.println("Counter = " + dfs.getCounter() + ";t="+
-        dfs.getToken());
     assertEquals("renew wasn't called as many times as expected(4):",
-        numberOfExpectedRenewals, dfs.getCounter());
-    assertEquals("most recently renewed token mismatch", dfs.getToken(), 
+        numberOfExpectedRenewals, Renewer.counter);
+    assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, 
         token1);
     
     // Test 2. 
@@ -277,8 +296,8 @@ public class TestDelegationTokenRenewal 
     MyToken token4 = dfs.getDelegationToken(new Text("user4"));
     
     //to cause this one to be set for renew in 2 secs
-    dfs.setTokenToRenewIn2Sec(token4); 
-    System.out.println("token="+token4+" should be renewed for 2 secs");
+    Renewer.tokenToRenewIn2Sec = token4; 
+    LOG.info("token="+token4+" should be renewed for 2 secs");
     
     String nn4 = DelegationTokenRenewal.SCHEME + "://host4:0";
     ts.addToken(new Text(nn4), token4);
@@ -287,24 +306,23 @@ public class TestDelegationTokenRenewal 
     JobID jid2 = new JobID("job2",1);
     DelegationTokenRenewal.registerDelegationTokensForRenewal(jid2, ts, conf);
     DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jid2);
-    numberOfExpectedRenewals = dfs.getCounter(); // number of renewals so far
+    numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
     try {
       Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew
     } catch (InterruptedException e) {}
-    System.out.println("Counter = " + dfs.getCounter() + ";t="+dfs.getToken());
+    System.out.println("Counter = " + Renewer.counter + ";t="+ 
+                       Renewer.lastRenewed);
     
     // counter and the token should stil be the old ones
     assertEquals("renew wasn't called as many times as expected",
-        numberOfExpectedRenewals, dfs.getCounter());
+        numberOfExpectedRenewals, Renewer.counter);
     
     // also renewing of the cancelled token should fail
-    boolean exception=false;
     try {
-      dfs.renewDelegationToken(token4);
+      token4.renew(conf);
+      fail("Renew of canceled token didn't fail");
     } catch (InvalidToken ite) {
       //expected
-      exception = true;
     }
-    assertTrue("Renew of canceled token didn't fail", exception);
   }
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java Wed Nov  2 05:34:31 2011
@@ -16,12 +16,9 @@
  */
 package org.apache.hadoop.mapreduce.security.token.delegation;
 
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
 import java.security.PrivilegedExceptionAction;
 
 import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
@@ -32,6 +29,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
@@ -51,6 +49,7 @@ public class TestDelegationToken {
   }
   
   @Test
+  @Ignore
   public void testDelegationToken() throws Exception {
     
     JobClient client;

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java Wed Nov  2 05:34:31 2011
@@ -107,11 +107,13 @@ public class MapredTestDriver {
   }
 
   public void run(String argv[]) {
+    int exitCode = -1;
     try {
-      pgd.driver(argv);
+      exitCode = pgd.driver(argv);
     } catch(Throwable e) {
       e.printStackTrace();
     }
+    System.exit(exitCode);
   }
 
   public static void main(String argv[]){

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Wed Nov  2 05:34:31 2011
@@ -246,8 +246,57 @@ public class TestRumenJobTraces {
   }
 
   /**
-   * Tests if {@link TraceBuilder} can correctly identify and parse jobhistory
-   * filenames. The testcase checks if {@link TraceBuilder}
+   * Validate the parsing of given history file name. Also validate the history
+   * file name suffixed with old/stale file suffix.
+   * @param jhFileName job history file path
+   * @param jid JobID
+   */
+  private void validateHistoryFileNameParsing(Path jhFileName,
+      org.apache.hadoop.mapred.JobID jid) {
+    JobID extractedJID =
+      JobID.forName(JobHistoryUtils.extractJobID(jhFileName.getName()));
+    assertEquals("TraceBuilder failed to parse the current JH filename"
+                 + jhFileName, jid, extractedJID);
+    // test jobhistory filename with old/stale file suffix
+    jhFileName = jhFileName.suffix(JobHistory.getOldFileSuffix("123"));
+    extractedJID =
+      JobID.forName(JobHistoryUtils.extractJobID(jhFileName.getName()));
+    assertEquals("TraceBuilder failed to parse the current JH filename"
+                 + "(old-suffix):" + jhFileName,
+                 jid, extractedJID);
+  }
+
+  /**
+   * Validate the parsing of given history conf file name. Also validate the
+   * history conf file name suffixed with old/stale file suffix.
+   * @param jhConfFileName job history conf file path
+   * @param jid JobID
+   */
+  private void validateJHConfFileNameParsing(Path jhConfFileName,
+      org.apache.hadoop.mapred.JobID jid) {
+    assertTrue("TraceBuilder failed to parse the JH conf filename:"
+               + jhConfFileName,
+               JobHistoryUtils.isJobConfXml(jhConfFileName.getName()));
+    JobID extractedJID =
+      JobID.forName(JobHistoryUtils.extractJobID(jhConfFileName.getName()));
+    assertEquals("TraceBuilder failed to parse the current JH conf filename:"
+                 + jhConfFileName, jid, extractedJID);
+    // Test jobhistory conf filename with old/stale file suffix
+    jhConfFileName = jhConfFileName.suffix(JobHistory.getOldFileSuffix("123"));
+    assertTrue("TraceBuilder failed to parse the current JH conf filename"
+               + " (old suffix):" + jhConfFileName,
+               JobHistoryUtils.isJobConfXml(jhConfFileName.getName()));
+    extractedJID =
+      JobID.forName(JobHistoryUtils.extractJobID(jhConfFileName.getName()));
+    assertEquals("TraceBuilder failed to parse the JH conf filename"
+                 + "(old-suffix):" + jhConfFileName,
+                 jid, extractedJID);
+  }
+
+  /**
+   * Tests if {@link TraceBuilder} can correctly identify and parse different
+   * versions of jobhistory filenames. The testcase checks if
+   * {@link TraceBuilder}
    *   - correctly identifies a jobhistory filename without suffix
    *   - correctly parses a jobhistory filename without suffix to extract out 
    *     the jobid
@@ -261,36 +310,37 @@ public class TestRumenJobTraces {
   public void testJobHistoryFilenameParsing() throws IOException {
     final Configuration conf = new Configuration();
     final FileSystem lfs = FileSystem.getLocal(conf);
-    String user = "test";
+
     org.apache.hadoop.mapred.JobID jid = 
       new org.apache.hadoop.mapred.JobID("12345", 1);
     final Path rootInputDir =
       new Path(System.getProperty("test.tools.input.dir", ""))
             .makeQualified(lfs.getUri(), lfs.getWorkingDirectory());
     
-    // Check if jobhistory filename are detected properly
-    Path jhFilename = JobHistory.getJobHistoryFile(rootInputDir, jid, user);
-    JobID extractedJID = 
-      JobID.forName(TraceBuilder.extractJobID(jhFilename.getName()));
-    assertEquals("TraceBuilder failed to parse the current JH filename", 
-                 jid, extractedJID);
-    // test jobhistory filename with old/stale file suffix
-    jhFilename = jhFilename.suffix(JobHistory.getOldFileSuffix("123"));
-    extractedJID =
-      JobID.forName(TraceBuilder.extractJobID(jhFilename.getName()));
-    assertEquals("TraceBuilder failed to parse the current JH filename"
-                 + "(old-suffix)", 
-                 jid, extractedJID);
-    
-    // Check if the conf filename in jobhistory are detected properly
+    // Check if current jobhistory filenames are detected properly
+    Path jhFilename = org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
+        .getStagingJobHistoryFile(rootInputDir, jid.toString(), 1);
+    validateHistoryFileNameParsing(jhFilename, jid);
+
+    // Check if Pre21 V1 jophistory file names are detected properly
+    jhFilename = new Path("jt-identifier_" + jid + "_user-name_job-name");
+    validateHistoryFileNameParsing(jhFilename, jid);
+
+    // Check if Pre21 V2 jobhistory file names are detected properly
+    jhFilename = new Path(jid + "_user-name_job-name");
+    validateHistoryFileNameParsing(jhFilename, jid);
+
+    // Check if the current jobhistory conf filenames are detected properly
     Path jhConfFilename = JobHistory.getConfFile(rootInputDir, jid);
-    assertTrue("TraceBuilder failed to parse the current JH conf filename", 
-               TraceBuilder.isJobConfXml(jhConfFilename.getName(), null));
-    // test jobhistory conf filename with old/stale file suffix
-    jhConfFilename = jhConfFilename.suffix(JobHistory.getOldFileSuffix("123"));
-    assertTrue("TraceBuilder failed to parse the current JH conf filename" 
-               + " (old suffix)", 
-               TraceBuilder.isJobConfXml(jhConfFilename.getName(), null));
+    validateJHConfFileNameParsing(jhConfFilename, jid);
+
+    // Check if Pre21 V1 jobhistory conf file names are detected properly
+    jhConfFilename = new Path("jt-identifier_" + jid + "_conf.xml");
+    validateJHConfFileNameParsing(jhConfFilename, jid);
+
+    // Check if Pre21 V2 jobhistory conf file names are detected properly
+    jhConfFilename = new Path(jid + "_conf.xml");
+    validateJHConfFileNameParsing(jhConfFilename, jid);
   }
 
   /**

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java Wed Nov  2 05:34:31 2011
@@ -1323,13 +1323,11 @@ public class HadoopLogsAnalyzer extends 
       int distance = Integer.MAX_VALUE;
 
       if (hostName != null) {
-        attempt.setHostName(hostName);
 
-        ParsedHost host = null;
-
-        host = getAndRecordParsedHost(hostName);
+        ParsedHost host = getAndRecordParsedHost(hostName);
 
         if (host != null) {
+          attempt.setHostName(host.getNodeName(), host.getRackName());
           attempt.setLocation(host.makeLoggedLocation());
         }
 
@@ -1492,8 +1490,10 @@ public class HadoopLogsAnalyzer extends 
           failedReduceAttemptTimes.enter(runtime);
         }
       }
-      if (hostName != null) {
-        attempt.setHostName(hostName);
+
+      ParsedHost host = getAndRecordParsedHost(hostName);
+      if (host != null) {
+        attempt.setHostName(host.getNodeName(), host.getRackName());
       }
 
       if (attemptID != null) {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Wed Nov  2 05:34:31 2011
@@ -28,6 +28,7 @@ import java.util.regex.Pattern;
 
 import org.apache.hadoop.mapred.TaskStatus;
 import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent;
 import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
 import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent;
 import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
@@ -129,7 +130,11 @@ public class JobBuilder {
     }
 
     // these are in lexicographical order by class name.
-    if (event instanceof JobFinishedEvent) {
+    if (event instanceof AMStartedEvent) {
+      // ignore this event as Rumen currently doesnt need this event
+      //TODO Enhance Rumen to process this event and capture restarts
+      return;
+    } else if (event instanceof JobFinishedEvent) {
       processJobFinishedEvent((JobFinishedEvent) event);
     } else if (event instanceof JobInfoChangeEvent) {
       processJobInfoChangeEvent((JobInfoChangeEvent) event);
@@ -517,7 +522,8 @@ public class JobBuilder {
       return;
     }
     attempt.setResult(getPre21Value(event.getTaskStatus()));
-    attempt.setHostName(event.getHostname());
+    attempt.setHostName(event.getHostname(), event.getRackName());
+
     // XXX There may be redundant location info available in the event.
     // We might consider extracting it from this event. Currently this
     // is redundant, but making this will add future-proofing.
@@ -540,7 +546,8 @@ public class JobBuilder {
       return;
     }
     attempt.setResult(getPre21Value(event.getTaskStatus()));
-    attempt.setHostName(event.getHostname());
+    attempt.setHostName(event.getHostname(), event.getRackname());
+
     // XXX There may be redundant location info available in the event.
     // We might consider extracting it from this event. Currently this
     // is redundant, but making this will add future-proofing.

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java Wed Nov  2 05:34:31 2011
@@ -328,8 +328,11 @@ public class LoggedTaskAttempt implement
     return hostName;
   }
 
-  void setHostName(String hostName) {
-    this.hostName = hostName == null ? null : hostName.intern();
+
+  // hostName is saved in the format rackName/NodeName
+  void setHostName(String hostName, String rackName) {
+    this.hostName = hostName == null || rackName == null ? null
+        : rackName.intern() + "/" + hostName.intern();
   }
 
   public long getHdfsBytesRead() {

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java Wed Nov  2 05:34:31 2011
@@ -73,7 +73,7 @@ public class MapAttempt20LineHistoryEven
               that.originalTaskType, status,
              Long.parseLong(finishTime),
              Long.parseLong(finishTime),
-             hostName, state, maybeParseCounters(counters),
+             hostName, null, state, maybeParseCounters(counters),
              null);
         }
       }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Pre21JobHistoryConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Pre21JobHistoryConstants.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Pre21JobHistoryConstants.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/Pre21JobHistoryConstants.java Wed Nov  2 05:34:31 2011
@@ -20,10 +20,10 @@ package org.apache.hadoop.tools.rumen;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
 
 /**
- * 
- *
+ * Job History related constants for Hadoop releases prior to 0.21
  */
 public class Pre21JobHistoryConstants {
   
@@ -51,18 +51,34 @@ public class Pre21JobHistoryConstants {
   }
   
   /**
-   * Pre21 regex for jobhistory filename 
+   * Regex for Pre21 V1(old) jobhistory filename
    *   i.e jt-identifier_job-id_user-name_job-name
    */
-  static final Pattern JOBHISTORY_FILENAME_REGEX =
+  static final Pattern JOBHISTORY_FILENAME_REGEX_V1 =
     Pattern.compile("[^.].+_(" + JobID.JOBID_REGEX + ")_.+");
+  /**
+   * Regex for Pre21 V2(new) jobhistory filename
+   *   i.e job-id_user-name_job-name
+   */
+  static final Pattern JOBHISTORY_FILENAME_REGEX_V2 =
+    Pattern.compile("(" + JobID.JOBID_REGEX + ")_.+");
+
+  static final String OLD_FULL_SUFFIX_REGEX_STRING =
+    "(?:\\.[0-9]+" + Pattern.quote(JobHistory.OLD_SUFFIX) + ")";
 
   /**
-   * Pre21 regex for jobhistory conf filename 
+   * Regex for Pre21 V1(old) jobhistory conf filename 
    *   i.e jt-identifier_job-id_conf.xml
    */
-  static final Pattern CONF_FILENAME_REGEX =
-    Pattern.compile("[^.].+_(" + JobID.JOBID_REGEX 
-                    + ")_conf.xml(?:\\.[0-9a-zA-Z]+)?");
+  static final Pattern CONF_FILENAME_REGEX_V1 =
+    Pattern.compile("[^.].+_(" + JobID.JOBID_REGEX + ")_conf.xml"
+                    + OLD_FULL_SUFFIX_REGEX_STRING + "?");
+  /**
+   * Regex for Pre21 V2(new) jobhistory conf filename
+   *   i.e job-id_conf.xml
+   */
+  static final Pattern CONF_FILENAME_REGEX_V2 =
+    Pattern.compile("(" + JobID.JOBID_REGEX + ")_conf.xml"
+                    + OLD_FULL_SUFFIX_REGEX_STRING + "?");
  
 }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java Wed Nov  2 05:34:31 2011
@@ -77,7 +77,7 @@ public class ReduceAttempt20LineHistoryE
              Long.parseLong(shuffleFinish),
              Long.parseLong(sortFinish),
              Long.parseLong(finishTime),
-             hostName,
+             hostName, null,
              state, maybeParseCounters(counters),
              null);
         }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java Wed Nov  2 05:34:31 2011
@@ -79,7 +79,7 @@ public abstract class TaskAttempt20LineE
                 .parseInt(httpPort);
 
         return new TaskAttemptStartedEvent(taskAttemptID,
-            that.originalTaskType, that.originalStartTime, trackerName, port);
+            that.originalTaskType, that.originalStartTime, trackerName, port, -1);
       }
 
       return null;

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java Wed Nov  2 05:34:31 2011
@@ -198,42 +198,6 @@ public class TraceBuilder extends Config
     }
   }
 
-  private static String applyParser(String fileName, Pattern pattern) {
-    Matcher matcher = pattern.matcher(fileName);
-
-    if (!matcher.matches()) {
-      return null;
-    }
-
-    return matcher.group(1);
-  }
-
-  /**
-   * @param fileName
-   * @return the jobID String, parsed out of the file name. We return a valid
-   *         String for either a history log file or a config file. Otherwise,
-   *         [especially for .crc files] we return null.
-   */
-  static String extractJobID(String fileName) {
-    String jobId = applyParser(fileName, JobHistory.JOBHISTORY_FILENAME_REGEX);
-    if (jobId == null) {
-      // check if its a pre21 jobhistory file
-      jobId = applyParser(fileName, 
-                          Pre21JobHistoryConstants.JOBHISTORY_FILENAME_REGEX);
-    }
-    return jobId;
-  }
-
-  static boolean isJobConfXml(String fileName, InputStream input) {
-    String jobId = applyParser(fileName, JobHistory.CONF_FILENAME_REGEX);
-    if (jobId == null) {
-      // check if its a pre21 jobhistory conf file
-      jobId = applyParser(fileName, 
-                          Pre21JobHistoryConstants.CONF_FILENAME_REGEX);
-    }
-    return jobId != null;
-  }
-
 
   @SuppressWarnings("unchecked")
   @Override
@@ -268,7 +232,7 @@ public class TraceBuilder extends Config
             JobHistoryParser parser = null;
 
             try {
-              String jobID = extractJobID(filePair.first());
+              String jobID = JobHistoryUtils.extractJobID(filePair.first());
               if (jobID == null) {
                 LOG.warn("File skipped: Invalid file name: "
                     + filePair.first());
@@ -282,8 +246,9 @@ public class TraceBuilder extends Config
                 jobBuilder = new JobBuilder(jobID);
               }
 
-              if (isJobConfXml(filePair.first(), ris)) {
-            	processJobConf(JobConfigurationParser.parse(ris.rewind()), jobBuilder);
+              if (JobHistoryUtils.isJobConfXml(filePair.first())) {
+                processJobConf(JobConfigurationParser.parse(ris.rewind()),
+                               jobBuilder);
               } else {
                 parser = JobHistoryParserFactory.getParser(ris);
                 if (parser == null) {

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
 /hadoop/core/trunk/src/webapps/job:776175-785643



Mime
View raw message