hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gkesa...@apache.org
Subject svn commit: r1369164 [3/7] - in /hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./ conf/ dev-support/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduc...
Date Fri, 03 Aug 2012 19:00:51 GMT
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java Fri Aug  3 19:00:15 2012
@@ -24,9 +24,11 @@ import static org.apache.hadoop.mapreduc
 import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_LOG_TYPE;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.NM_NODENAME;
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.verify;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -35,6 +37,7 @@ import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.MockJobs;
@@ -44,13 +47,14 @@ import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.ClusterInfo;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.webapp.log.AggregatedLogsPage;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
 import org.junit.Test;
 
-import static org.mockito.Mockito.verify;
+import com.google.inject.AbstractModule;
 import com.google.inject.Injector;
 
 public class TestHSWebApp {
@@ -251,6 +255,64 @@ public class TestHSWebApp {
         "Aggregation is not enabled. Try the nodemanager at "
             + MockJobs.NM_HOST + ":" + MockJobs.NM_PORT);
   }
+
+  @Test
+  public void testLogsViewSingle() throws IOException {
+    LOG.info("HsLogsPage with params for single log and data limits");
+    TestAppContext ctx = new TestAppContext();
+    Map<String, String> params = new HashMap<String, String>();
+
+    final Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+
+    params.put("start", "-2048");
+    params.put("end", "-1024");
+    params.put(CONTAINER_LOG_TYPE, "syslog");
+    params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
+        .toString());
+    params.put(NM_NODENAME,
+        BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
+    params.put(ENTITY_STRING, "container_10_0001_01_000001");
+    params.put(APP_OWNER, "owner");
+
+    Injector injector =
+        WebAppTests.testPage(AggregatedLogsPage.class, AppContext.class, ctx,
+            params, new AbstractModule() {
+          @Override
+          protected void configure() {
+            bind(Configuration.class).toInstance(conf);
+          }
+        });
+    PrintWriter spyPw = WebAppTests.getPrintWriter(injector);
+    verify(spyPw).write(
+        "Logs not available for container_10_0001_01_000001."
+            + " Aggregation may not be complete, "
+            + "Check back later or try the nodemanager at "
+            + MockJobs.NM_HOST + ":" + MockJobs.NM_PORT);
+  }
+
+  @Test
+  public void testLogsViewBadStartEnd() throws IOException {
+    LOG.info("HsLogsPage with bad start/end params");
+    TestAppContext ctx = new TestAppContext();
+    Map<String, String> params = new HashMap<String, String>();
+
+    params.put("start", "foo");
+    params.put("end", "bar");
+    params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
+        .toString());
+    params.put(NM_NODENAME,
+        BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
+    params.put(ENTITY_STRING, "container_10_0001_01_000001");
+    params.put(APP_OWNER, "owner");
+
+    Injector injector =
+        WebAppTests.testPage(AggregatedLogsPage.class, AppContext.class, ctx,
+            params);
+    PrintWriter spyPw = WebAppTests.getPrintWriter(injector);
+    verify(spyPw).write("Invalid log start value: foo");
+    verify(spyPw).write("Invalid log end value: bar");
+  }
 }
   
  

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml Fri Aug  3 19:00:15 2012
@@ -99,6 +99,23 @@
     </dependency>
   </dependencies>
 
+ <profiles>
+  <profile>
+    <id>clover</id>
+    <activation>
+      <activeByDefault>false</activeByDefault>
+      <property>
+        <name>clover</name>
+      </property>
+    </activation>
+    <dependencies>
+      <dependency>
+        <groupId>com.cenqua.clover</groupId>
+        <artifactId>clover</artifactId>
+      </dependency>
+    </dependencies>
+  </profile>
+</profiles>
   <build>
     <plugins>
       <plugin>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Fri Aug  3 19:00:15 2012
@@ -232,8 +232,9 @@ public class ClientServiceDelegate {
     if (user == null) {
       throw RPCUtil.getRemoteException("User is not set in the application report");
     }
-    if (application.getYarnApplicationState() == YarnApplicationState.NEW ||
-        application.getYarnApplicationState() == YarnApplicationState.SUBMITTED) {
+    if (application.getYarnApplicationState() == YarnApplicationState.NEW
+        || application.getYarnApplicationState() == YarnApplicationState.SUBMITTED
+        || application.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
       realProxy = null;
       return getNotRunningJob(application, JobState.NEW);
     }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Fri Aug  3 19:00:15 2012
@@ -56,6 +56,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -75,13 +76,17 @@ public class NotRunningJob implements MR
 
 
   private ApplicationReport getUnknownApplicationReport() {
-    ApplicationId unknownAppId = recordFactory.newRecordInstance(ApplicationId.class);
+    ApplicationId unknownAppId = recordFactory
+        .newRecordInstance(ApplicationId.class);
+    ApplicationAttemptId unknownAttemptId = recordFactory
+        .newRecordInstance(ApplicationAttemptId.class);
 
-    // Setting AppState to NEW and finalStatus to UNDEFINED as they are never used 
+    // Setting AppState to NEW and finalStatus to UNDEFINED as they are never
+    // used
     // for a non running job
-    return BuilderUtils.newApplicationReport(unknownAppId, "N/A", "N/A", "N/A", "N/A", 0, "", 
-        YarnApplicationState.NEW, "N/A", "N/A", 0, 0, 
-        FinalApplicationStatus.UNDEFINED, null, "N/A");    
+    return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
+        "N/A", "N/A", "N/A", "N/A", 0, "", YarnApplicationState.NEW, "N/A",
+        "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A");
   }
 
   NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java Fri Aug  3 19:00:15 2012
@@ -210,19 +210,10 @@ public class MRCaching {
     fs.copyFromLocalFile(tarPath1, cachePath);
     fs.copyFromLocalFile(tarPath2, cachePath);
   }
- 
-  public static TestResult launchMRCache(String indir,
-                                         String outdir, String cacheDir, 
-                                         JobConf conf, String input) 
-  throws IOException {
-    setupCache(cacheDir, FileSystem.get(conf));
-    return launchMRCache(indir,outdir, cacheDir, conf, input, false); 
-  }
   
   public static TestResult launchMRCache(String indir,
                                          String outdir, String cacheDir, 
-                                         JobConf conf, String input,
-                                         boolean withSymlink)
+                                         JobConf conf, String input)
     throws IOException {
     String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/tmp"))
       .toString().replace(' ', '+');
@@ -256,24 +247,13 @@ public class MRCaching {
     conf.setNumReduceTasks(1);
     conf.setSpeculativeExecution(false);
     URI[] uris = new URI[6];
-    if (!withSymlink) {
-      conf.setMapperClass(MRCaching.MapClass.class);
-      uris[0] = fs.getUri().resolve(cacheDir + "/test.txt");
-      uris[1] = fs.getUri().resolve(cacheDir + "/test.jar");
-      uris[2] = fs.getUri().resolve(cacheDir + "/test.zip");
-      uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz");
-      uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz");
-      uris[5] = fs.getUri().resolve(cacheDir + "/test.tar");
-    } else {
-      DistributedCache.createSymlink(conf);
-      conf.setMapperClass(MRCaching.MapClass2.class);
-      uris[0] = fs.getUri().resolve(cacheDir + "/test.txt#" + "test.txt");
-      uris[1] = fs.getUri().resolve(cacheDir + "/test.jar#" + "testjar");
-      uris[2] = fs.getUri().resolve(cacheDir + "/test.zip#" + "testzip");
-      uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz#" + "testtgz");
-      uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz#" + "testtargz");
-      uris[5] = fs.getUri().resolve(cacheDir + "/test.tar#" + "testtar");
-    }
+    conf.setMapperClass(MRCaching.MapClass2.class);
+    uris[0] = fs.getUri().resolve(cacheDir + "/test.txt");
+    uris[1] = fs.getUri().resolve(cacheDir + "/test.jar");
+    uris[2] = fs.getUri().resolve(cacheDir + "/test.zip");
+    uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz");
+    uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz");
+    uris[5] = fs.getUri().resolve(cacheDir + "/test.tar");
     DistributedCache.addCacheFile(uris[0], conf);
 
     // Save expected file sizes

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Fri Aug  3 19:00:15 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.lib.HashPartitioner;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.fs.*;
@@ -345,7 +344,8 @@ public class SortValidator extends Confi
 
       FileInputFormat.setInputPaths(jobConf, sortInput);
       FileInputFormat.addInputPath(jobConf, sortOutput);
-      Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
+      Path outputPath = new Path(new Path("/tmp",
+           "sortvalidate"), UUID.randomUUID().toString());
       if (defaultfs.exists(outputPath)) {
         defaultfs.delete(outputPath, true);
       }
@@ -365,31 +365,44 @@ public class SortValidator extends Confi
       Date startTime = new Date();
       System.out.println("Job started: " + startTime);
       JobClient.runJob(jobConf);
-      Date end_time = new Date();
-      System.out.println("Job ended: " + end_time);
-      System.out.println("The job took " + 
-                         (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
-      
-      // Check to ensure that the statistics of the 
-      // framework's sort-input and sort-output match
-      SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
-                                                          new Path(outputPath, "part-00000"), defaults);
-      IntWritable k1 = new IntWritable();
-      IntWritable k2 = new IntWritable();
-      RecordStatsWritable v1 = new RecordStatsWritable();
-      RecordStatsWritable v2 = new RecordStatsWritable();
-      if (!stats.next(k1, v1)) {
-        throw new IOException("Failed to read record #1 from reduce's output");
-      }
-      if (!stats.next(k2, v2)) {
-        throw new IOException("Failed to read record #2 from reduce's output");
-      }
-
-      if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != v2.getRecords()) || 
-          v1.getChecksum() != v2.getChecksum()) {
-        throw new IOException("(" + 
-                              v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() + ") v/s (" +
-                              v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() + ")");
+      try {
+        Date end_time = new Date();
+        System.out.println("Job ended: " + end_time);
+        System.out.println("The job took " + 
+            (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
+
+        // Check to ensure that the statistics of the 
+        // framework's sort-input and sort-output match
+        SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
+            new Path(outputPath, "part-00000"), defaults);
+        try {
+          IntWritable k1 = new IntWritable();
+          IntWritable k2 = new IntWritable();
+          RecordStatsWritable v1 = new RecordStatsWritable();
+          RecordStatsWritable v2 = new RecordStatsWritable();
+          if (!stats.next(k1, v1)) {
+            throw new IOException(
+                "Failed to read record #1 from reduce's output");
+          }
+          if (!stats.next(k2, v2)) {
+            throw new IOException(
+                "Failed to read record #2 from reduce's output");
+          }
+
+          if ((v1.getBytes() != v2.getBytes()) || 
+              (v1.getRecords() != v2.getRecords()) || 
+              v1.getChecksum() != v2.getChecksum()) {
+            throw new IOException("(" + 
+                v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum()
+                + ") v/s (" +
+                v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum()
+                + ")");
+          }
+        } finally {
+          stats.close();
+        }
+      } finally {
+        defaultfs.delete(outputPath, true);
       }
     }
 

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Fri Aug  3 19:00:15 2012
@@ -50,6 +50,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.apache.hadoop.mapreduce.v2.api.records.JobState;
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -404,17 +405,23 @@ public class TestClientServiceDelegate {
   }
 
   private ApplicationReport getFinishedApplicationReport() {
-    return BuilderUtils.newApplicationReport(BuilderUtils.newApplicationId(
-        1234, 5), "user", "queue", "appname", "host", 124, null,
-        YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A");
+    ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
+    ApplicationAttemptId attemptId = BuilderUtils.newApplicationAttemptId(
+        appId, 0);
+    return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
+        "appname", "host", 124, null, YarnApplicationState.FINISHED,
+        "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
+        "N/A");
   }
 
   private ApplicationReport getRunningApplicationReport(String host, int port) {
-    return BuilderUtils.newApplicationReport(BuilderUtils.newApplicationId(
-        1234, 5), "user", "queue", "appname", host, port, null,
-        YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.UNDEFINED, null, "N/A");
+    ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
+    ApplicationAttemptId attemptId = BuilderUtils.newApplicationAttemptId(
+        appId, 0);
+    return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
+        "appname", host, port, null, YarnApplicationState.RUNNING,
+        "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null,
+        "N/A");
   }
 
   private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java Fri Aug  3 19:00:15 2012
@@ -97,13 +97,16 @@ public class TestFileInputFormat extends
   private void createInputs(FileSystem fs, Path inDir, String fileName)
   throws IOException {
     // create a multi-block file on hdfs
-    DataOutputStream out = fs.create(new Path(inDir, fileName), true, 4096,
-                                     (short) 2, 512, null);
+    Path path = new Path(inDir, fileName);
+    final short replication = 2;
+    DataOutputStream out = fs.create(path, true, 4096,
+                                     replication, 512, null);
     for(int i=0; i < 1000; ++i) {
       out.writeChars("Hello\n");
     }
     out.close();
     System.out.println("Wrote file");
+    DFSTestUtil.waitReplication(fs, path, replication);
   }
 
   public void testNumInputs() throws Exception {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java Fri Aug  3 19:00:15 2012
@@ -48,7 +48,7 @@ public class TestMiniMRDFSCaching extend
                                             "/cachedir",
                                             mr.createJobConf(),
                                             "The quick brown fox\nhas many silly\n"
-                                            + "red fox sox\n", false);
+                                            + "red fox sox\n");
       assertTrue("Archives not matching", ret.isOutputOk);
       // launch MR cache with symlinks
       ret = MRCaching.launchMRCache("/testing/wc/input",
@@ -56,7 +56,7 @@ public class TestMiniMRDFSCaching extend
                                     "/cachedir",
                                     mr.createJobConf(),
                                     "The quick brown fox\nhas many silly\n"
-                                    + "red fox sox\n", true);
+                                    + "red fox sox\n");
       assertTrue("Archives not matching", ret.isOutputOk);
     } finally {
       if (fileSys != null) {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Fri Aug  3 19:00:15 2012
@@ -113,10 +113,6 @@ public class MiniMRYarnCluster extends M
     // for corresponding uberized tests.
     conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
 
-    // Set config for JH Server
-    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS,
-        JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
-
     super.init(conf);
   }
 
@@ -128,10 +124,15 @@ public class MiniMRYarnCluster extends M
     @Override
     public synchronized void start() {
       try {
-        getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
-                        MiniYARNCluster.getHostname() + ":0");
-        getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
-                        MiniYARNCluster.getHostname() + ":0");
+        if (!getConfig().getBoolean(
+            JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
+            JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
+          // pick free random ports.
+          getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+          getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+        }
         historyServer = new JobHistoryServer();
         historyServer.init(getConfig());
         new Thread() {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Fri Aug  3 19:00:15 2012
@@ -211,6 +211,7 @@ public class TestMRJobs {
     Path outputDir =
         new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output");
     FileOutputFormat.setOutputPath(job, outputDir);
+    job.setSpeculativeExecution(false);
     job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
     job.setJarByClass(RandomTextWriterJob.class);
     job.setMaxMapAttempts(1); // speed up failures
@@ -462,7 +463,6 @@ public class TestMRJobs {
     job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
     job.addArchiveToClassPath(third);
     job.addCacheArchive(fourth.toUri());
-    job.createSymlink();
     job.setMaxMapAttempts(1); // speed up failures
 
     job.submit();

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java Fri Aug  3 19:00:15 2012
@@ -301,7 +301,6 @@ public class TestSpeculativeExecution {
 
     // Creates the Job Configuration
     job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.createSymlink();
     job.setMaxMapAttempts(2);
 
     job.submit();

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java Fri Aug  3 19:00:15 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.mapred.TestSequ
 import org.apache.hadoop.mapred.TestTextInputFormat;
 import org.apache.hadoop.mapred.ThreadedMapBenchmark;
 import org.apache.hadoop.mapreduce.FailJob;
+import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.util.ProgramDriver;
 
@@ -101,6 +102,8 @@ public class MapredTestDriver {
           "Job History Log analyzer.");
       pgd.addClass(SliveTest.class.getSimpleName(), SliveTest.class, 
           "HDFS Stress Test and Live Data Verification.");
+      pgd.addClass("minicluster", MiniHadoopClusterManager.class,
+      "Single process HDFS and MR cluster.");
     } catch(Throwable e) {
       e.printStackTrace();
     }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Fri Aug  3 19:00:15 2012
@@ -55,7 +55,10 @@ import org.apache.hadoop.fs.LocalDirAllo
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader;
@@ -84,9 +87,7 @@ import org.jboss.netty.channel.ChannelHa
 import org.jboss.netty.channel.ChannelPipeline;
 import org.jboss.netty.channel.ChannelPipelineFactory;
 import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.DefaultFileRegion;
 import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.FileRegion;
 import org.jboss.netty.channel.MessageEvent;
 import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
 import org.jboss.netty.channel.group.ChannelGroup;
@@ -101,6 +102,7 @@ import org.jboss.netty.handler.codec.htt
 import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.jboss.netty.handler.codec.http.QueryStringDecoder;
+import org.jboss.netty.handler.ssl.SslHandler;
 import org.jboss.netty.handler.stream.ChunkedWriteHandler;
 import org.jboss.netty.util.CharsetUtil;
 
@@ -110,10 +112,27 @@ public class ShuffleHandler extends Abst
     implements AuxServices.AuxiliaryService {
 
   private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
+  
+  public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+  public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
+
+  public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+  public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
 
   private int port;
   private ChannelFactory selector;
   private final ChannelGroup accepted = new DefaultChannelGroup();
+  private HttpPipelineFactory pipelineFact;
+  private int sslFileBufferSize;
+
+  /**
+   * Should the shuffle use posix_fadvise calls to manage the OS cache during
+   * sendfile
+   */
+  private boolean manageOsCache;
+  private int readaheadLength;
+  private ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
+   
 
   public static final String MAPREDUCE_SHUFFLE_SERVICEID =
       "mapreduce.shuffle";
@@ -126,6 +145,11 @@ public class ShuffleHandler extends Abst
   public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
   public static final int DEFAULT_SHUFFLE_PORT = 8080;
 
+  public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
+    "mapreduce.shuffle.ssl.file.buffer.size";
+
+  public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
+
   @Metrics(about="Shuffle output metrics", context="mapred")
   static class ShuffleMetrics implements ChannelFutureListener {
     @Metric("Shuffle output in bytes")
@@ -231,6 +255,12 @@ public class ShuffleHandler extends Abst
 
   @Override
   public synchronized void init(Configuration conf) {
+    manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE,
+        DEFAULT_SHUFFLE_MANAGE_OS_CACHE);
+
+    readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES,
+        DEFAULT_SHUFFLE_READAHEAD_BYTES);
+    
     ThreadFactory bossFactory = new ThreadFactoryBuilder()
       .setNameFormat("ShuffleHandler Netty Boss #%d")
       .build();
@@ -249,7 +279,11 @@ public class ShuffleHandler extends Abst
   public synchronized void start() {
     Configuration conf = getConfig();
     ServerBootstrap bootstrap = new ServerBootstrap(selector);
-    HttpPipelineFactory pipelineFact = new HttpPipelineFactory(conf);
+    try {
+      pipelineFact = new HttpPipelineFactory(conf);
+    } catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
     bootstrap.setPipelineFactory(pipelineFact);
     port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
     Channel ch = bootstrap.bind(new InetSocketAddress(port));
@@ -259,6 +293,9 @@ public class ShuffleHandler extends Abst
     pipelineFact.SHUFFLE.setPort(port);
     LOG.info(getName() + " listening on port " + port);
     super.start();
+
+    sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
+                                    DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
   }
 
   @Override
@@ -266,6 +303,7 @@ public class ShuffleHandler extends Abst
     accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
     ServerBootstrap bootstrap = new ServerBootstrap(selector);
     bootstrap.releaseExternalResources();
+    pipelineFact.destroy();
     super.stop();
   }
 
@@ -283,22 +321,38 @@ public class ShuffleHandler extends Abst
   class HttpPipelineFactory implements ChannelPipelineFactory {
 
     final Shuffle SHUFFLE;
+    private SSLFactory sslFactory;
 
-    public HttpPipelineFactory(Configuration conf) {
+    public HttpPipelineFactory(Configuration conf) throws Exception {
       SHUFFLE = new Shuffle(conf);
+      if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+                          MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
+        sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+        sslFactory.init();
+      }
+    }
+
+    public void destroy() {
+      if (sslFactory != null) {
+        sslFactory.destroy();
+      }
     }
 
     @Override
     public ChannelPipeline getPipeline() throws Exception {
-        return Channels.pipeline(
-            new HttpRequestDecoder(),
-            new HttpChunkAggregator(1 << 16),
-            new HttpResponseEncoder(),
-            new ChunkedWriteHandler(),
-            SHUFFLE);
-        // TODO factor security manager into pipeline
-        // TODO factor out encode/decode to permit binary shuffle
-        // TODO factor out decode of index to permit alt. models
+      ChannelPipeline pipeline = Channels.pipeline();
+      if (sslFactory != null) {
+        pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
+      }
+      pipeline.addLast("decoder", new HttpRequestDecoder());
+      pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
+      pipeline.addLast("encoder", new HttpResponseEncoder());
+      pipeline.addLast("chunking", new ChunkedWriteHandler());
+      pipeline.addLast("shuffle", SHUFFLE);
+      return pipeline;
+      // TODO factor security manager into pipeline
+      // TODO factor out encode/decode to permit binary shuffle
+      // TODO factor out decode of index to permit alt. models
     }
 
   }
@@ -468,14 +522,14 @@ public class ShuffleHandler extends Abst
           base + "/file.out", conf);
       LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : " +
           indexFileName);
-      IndexRecord info = 
+      final IndexRecord info = 
         indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
       final ShuffleHeader header =
         new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce);
       final DataOutputBuffer dob = new DataOutputBuffer();
       header.write(dob);
       ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
-      File spillfile = new File(mapOutputFileName.toString());
+      final File spillfile = new File(mapOutputFileName.toString());
       RandomAccessFile spill;
       try {
         spill = new RandomAccessFile(spillfile, "r");
@@ -483,17 +537,28 @@ public class ShuffleHandler extends Abst
         LOG.info(spillfile + " not found");
         return null;
       }
-      final FileRegion partition = new DefaultFileRegion(
-          spill.getChannel(), info.startOffset, info.partLength);
-      ChannelFuture writeFuture = ch.write(partition);
-      writeFuture.addListener(new ChannelFutureListener() {
-          // TODO error handling; distinguish IO/connection failures,
-          //      attribute to appropriate spill output
+      ChannelFuture writeFuture;
+      if (ch.getPipeline().get(SslHandler.class) == null) {
+        final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
+            info.startOffset, info.partLength, manageOsCache, readaheadLength,
+            readaheadPool, spillfile.getAbsolutePath());
+        writeFuture = ch.write(partition);
+        writeFuture.addListener(new ChannelFutureListener() {
+            // TODO error handling; distinguish IO/connection failures,
+            //      attribute to appropriate spill output
           @Override
           public void operationComplete(ChannelFuture future) {
             partition.releaseExternalResources();
           }
         });
+      } else {
+        // HTTPS cannot be done with zero copy.
+        final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
+            info.startOffset, info.partLength, sslFileBufferSize,
+            manageOsCache, readaheadLength, readaheadPool,
+            spillfile.getAbsolutePath());
+        writeFuture = ch.write(chunk);
+      }
       metrics.shuffleConnections.incr();
       metrics.shuffleOutputBytes.incr(info.partLength); // optimistic
       return writeFuture;

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java Fri Aug  3 19:00:15 2012
@@ -167,7 +167,6 @@ public class Sort<K,V> extends Configure
       URI partitionUri = new URI(partitionFile.toString() +
                                  "#" + "_sortPartitioning");
       DistributedCache.addCacheFile(partitionUri, conf);
-      DistributedCache.createSymlink(conf);
     }
 
     System.out.println("Running on " +

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Fri Aug  3 19:00:15 2012
@@ -305,8 +305,7 @@ public class TeraSort extends Configured
         LOG.error(e.getMessage());
         return -1;
       }
-      job.addCacheFile(partitionUri);
-      job.createSymlink();    
+      job.addCacheFile(partitionUri);  
       long end = System.currentTimeMillis();
       System.out.println("Spent " + (end - start) + "ms computing partitions.");
       job.setPartitionerClass(TotalOrderPartitioner.class);

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml Fri Aug  3 19:00:15 2012
@@ -187,6 +187,11 @@
     <Class name="org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread" />
     <Bug pattern="DM_EXIT" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.YarnUncaughtExceptionHandler"/>
+    <Bug pattern="DM_EXIT"/>
+  </Match>
+
   <!-- AsyncDispatcher will kill the process if there is an error dispatching -->
   <Match>
     <Class name="org.apache.hadoop.yarn.event.AsyncDispatcher" />
@@ -231,4 +236,54 @@
     <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AppLogAggregatorImpl" />
     <Bug pattern="EI_EXPOSE_REP2" />
   </Match>
+
+
+   <!-- MAPREDUCE-4439 -->
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog" />
+      <Method name="shutdown" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="initialized" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="preemptionEnabled" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="queueMgr" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="sizeBasedWeight" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="userAsDefaultQueue" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="logDisabled" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="queueMaxAppsDefault" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+   <Match>
+     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+      <Field name="userMaxAppsDefault" />
+     <Bug pattern="IS2_INCONSISTENT_SYNC" />
+   </Match>
+
+
 </FindBugsFilter>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Fri Aug  3 19:00:15 2012
@@ -41,6 +41,12 @@ public interface ApplicationConstants {
    * only
    */
   public static final String AM_CONTAINER_ID_ENV = "AM_CONTAINER_ID";
+  
+  /**
+   * The environment variable for APPLICATION_ATTEMPT_ID. Set in AppMaster
+   * environment only
+   */
+  public static final String AM_APP_ATTEMPT_ID_ENV = "AM_APP_ATTEMPT_ID";
 
   /**
    * The environment variable for the NM_HOST. Set in the AppMaster environment

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Fri Aug  3 19:00:15 2012
@@ -60,6 +60,19 @@ public interface ApplicationReport {
   @Private
   @Unstable
   void setApplicationId(ApplicationId applicationId);
+  
+  /**
+   * Get the <code>ApplicationAttemptId</code> of the current
+   * attempt of the application
+   * @return <code>ApplicationAttemptId</code> of the attempt
+   */
+  @Private
+  @Unstable
+  ApplicationAttemptId getCurrentApplicationAttemptId();
+  
+  @Private
+  @Unstable
+  void setCurrentApplicationAttemptId(ApplicationAttemptId applicationAttemptId);
 
   /**
    * Get the <em>user</em> who submitted the application.

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Fri Aug  3 19:00:15 2012
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.api.records;
 
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -151,6 +150,28 @@ public interface ApplicationSubmissionCo
   @Public
   @Stable
   public void setAMContainerSpec(ContainerLaunchContext amContainer);
+  
+  /**
+   * Get if the RM should manage the execution of the AM. 
+   * If true, then the RM 
+   * will not allocate a container for the AM and start it. It will expect the 
+   * AM to be launched and connect to the RM within the AM liveliness period and 
+   * fail the app otherwise. The client should launch the AM only after the RM 
+   * has ACCEPTED the application and changed the <code>YarnApplicationState</code>.
+   * Such apps will not be retried by the RM on app attempt failure.
+   * The default value is false.
+   * @return true if the AM is not managed by the RM
+   */
+  @Public
+  @Unstable
+  public boolean getUnmanagedAM();
+  
+  /**
+   * @param value true if RM should not manage the AM
+   */
+  @Public
+  @Unstable
+  public void setUnmanagedAM(boolean value);
 
   /**
    * @return true if tokens should be canceled when the app completes.

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java Fri Aug  3 19:00:15 2012
@@ -50,11 +50,11 @@ public interface ContainerToken extends 
    */
   @Public
   @Stable
-  public abstract ByteBuffer getIdentifier();
+  ByteBuffer getIdentifier();
   
   @Private
   @Stable
-  public abstract void setIdentifier(ByteBuffer identifier);
+  void setIdentifier(ByteBuffer identifier);
 
   /**
    * Get the token password
@@ -62,11 +62,11 @@ public interface ContainerToken extends 
    */
   @Public
   @Stable
-  public abstract ByteBuffer getPassword();
+  ByteBuffer getPassword();
   
   @Private
   @Stable
-  public abstract void setPassword(ByteBuffer password);
+  void setPassword(ByteBuffer password);
 
   /**
    * Get the token kind.
@@ -74,11 +74,11 @@ public interface ContainerToken extends 
    */
   @Public
   @Stable
-  public abstract String getKind();
+  String getKind();
   
   @Private
   @Stable
-  public abstract void setKind(String kind);
+  void setKind(String kind);
 
   /**
    * Get the service to which the token is allocated.
@@ -86,10 +86,10 @@ public interface ContainerToken extends 
    */
   @Public
   @Stable
-  public abstract String getService();
+  String getService();
 
   @Private
   @Stable
-  public abstract void setService(String service);
+  void setService(String service);
 
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java Fri Aug  3 19:00:15 2012
@@ -32,7 +32,7 @@ public enum YarnApplicationState {
 
   /** Application which has been submitted. */
   SUBMITTED,
-
+  
   /** Application which is currently running. */
   RUNNING,
 
@@ -43,5 +43,8 @@ public enum YarnApplicationState {
   FAILED,
 
   /** Application which was terminated by a user or admin. */
-  KILLED
+  KILLED,
+
+  /** Application has been accepted by the scheduler */
+  ACCEPTED
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Fri Aug  3 19:00:15 2012
@@ -18,12 +18,14 @@
 
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.ProtoBase;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
@@ -39,6 +41,7 @@ implements ApplicationReport {
   boolean viaProto = false;
 
   ApplicationId applicationId;
+  ApplicationAttemptId currentApplicationAttemptId;
 
   public ApplicationReportPBImpl() {
     builder = ApplicationReportProto.newBuilder();
@@ -71,6 +74,20 @@ implements ApplicationReport {
     }
     builder.setAppResourceUsage(convertToProtoFormat(appInfo));
   }
+  
+  @Override
+  public ApplicationAttemptId getCurrentApplicationAttemptId() {
+    if (this.currentApplicationAttemptId != null) {
+      return this.currentApplicationAttemptId;
+    }
+
+    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasCurrentApplicationAttemptId()) {
+      return null;
+    }
+    this.currentApplicationAttemptId = convertFromProtoFormat(p.getCurrentApplicationAttemptId());
+    return this.currentApplicationAttemptId;
+  }
 
   @Override
   public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
@@ -199,6 +216,14 @@ implements ApplicationReport {
   }
 
   @Override
+  public void setCurrentApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
+    maybeInitBuilder();
+    if (applicationId == null)
+      builder.clearStatus();
+    this.currentApplicationAttemptId = applicationAttemptId;
+  }
+
+  @Override
   public void setTrackingUrl(String url) {
     maybeInitBuilder();
     if (url == null) {
@@ -330,6 +355,11 @@ implements ApplicationReport {
             builder.getApplicationId())) {
       builder.setApplicationId(convertToProtoFormat(this.applicationId));
     }
+    if (this.currentApplicationAttemptId != null
+        && !((ApplicationAttemptIdPBImpl) this.currentApplicationAttemptId).getProto().equals(
+            builder.getCurrentApplicationAttemptId())) {
+      builder.setCurrentApplicationAttemptId(convertToProtoFormat(this.currentApplicationAttemptId));
+    }
   }
 
   private void mergeLocalToProto() {
@@ -350,6 +380,10 @@ implements ApplicationReport {
   private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
     return ((ApplicationIdPBImpl) t).getProto();
   }
+  
+  private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) {
+    return ((ApplicationAttemptIdPBImpl) t).getProto();
+  }
 
   private ApplicationResourceUsageReport convertFromProtoFormat(ApplicationResourceUsageReportProto s) {
     return ProtoUtils.convertFromProtoFormat(s);
@@ -363,6 +397,11 @@ implements ApplicationReport {
       ApplicationIdProto applicationId) {
     return new ApplicationIdPBImpl(applicationId);
   }
+  
+  private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+      ApplicationAttemptIdProto applicationAttemptId) {
+    return new ApplicationAttemptIdPBImpl(applicationAttemptId);
+  }
 
   private YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto s) {
     return ProtoUtils.convertFromProtoFormat(s);

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Fri Aug  3 19:00:15 2012
@@ -208,12 +208,25 @@ implements ApplicationSubmissionContext 
   }
   
   @Override
+  public boolean getUnmanagedAM() {
+    ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+    //There is a default so cancelTokens should never be null
+    return p.getUnmanagedAm();
+  }
+  
+  @Override
+  public void setUnmanagedAM(boolean value) {
+    maybeInitBuilder();
+    builder.setUnmanagedAm(value);
+  }
+
+  @Override
   public boolean getCancelTokensWhenComplete() {
     ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
     //There is a default so cancelTokens should never be null
     return p.getCancelTokensWhenComplete();
   }
-
+  
   @Override
   public void setCancelTokensWhenComplete(boolean cancel) {
     maybeInitBuilder();

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Aug  3 19:00:15 2012
@@ -90,6 +90,7 @@ enum YarnApplicationStateProto {
   FINISHED = 4;
   FAILED = 5;
   KILLED = 6;
+  ACCEPTED = 7;
 }
 
 enum FinalApplicationStatusProto {
@@ -170,6 +171,7 @@ message ApplicationReportProto {
   optional FinalApplicationStatusProto final_application_status = 15;
   optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
   optional string originalTrackingUrl = 17;
+  optional ApplicationAttemptIdProto currentApplicationAttemptId = 18;
 }
 
 enum NodeStateProto {
@@ -235,6 +237,7 @@ message ApplicationSubmissionContextProt
   optional PriorityProto priority = 5;
   optional ContainerLaunchContextProto am_container_spec = 6;
   optional bool cancel_tokens_when_complete = 7 [default = true];
+  optional bool unmanaged_am = 8 [default = false];
 }
 
 enum ApplicationAccessTypeProto {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Aug  3 19:00:15 2012
@@ -290,7 +290,10 @@ public class ApplicationMaster {
     Map<String, String> envs = System.getenv();
 
     appAttemptID = Records.newRecord(ApplicationAttemptId.class);
-    if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
+    if (envs.containsKey(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)) {
+      appAttemptID = ConverterUtils.toApplicationAttemptId(envs
+          .get(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV));
+    } else if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
       if (cliParser.hasOption("app_attempt_id")) {
         String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
         appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Fri Aug  3 19:00:15 2012
@@ -508,8 +508,9 @@ public class Client {
     // For now setting all required classpaths including
     // the classpath to "." for the application jar
     StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
-    for (String c : conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)
-        .split(",")) {
+    for (String c : conf.getStrings(
+        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
       classPathEnv.append(':');
       classPathEnv.append(c.trim());
     }

Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Aug  3 19:00:15 2012
@@ -0,0 +1 @@
+target

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml Fri Aug  3 19:00:15 2012
@@ -30,5 +30,23 @@
 
   <modules>
     <module>hadoop-yarn-applications-distributedshell</module>
+    <module>hadoop-yarn-applications-unmanaged-am-launcher</module>
   </modules>
+ <profiles>
+  <profile>
+    <id>clover</id>
+    <activation>
+      <activeByDefault>false</activeByDefault>
+      <property>
+        <name>clover</name>
+      </property>
+    </activation>
+    <dependencies>
+      <dependency>
+        <groupId>com.cenqua.clover</groupId>
+        <artifactId>clover</artifactId>
+      </dependency>
+    </dependencies>
+  </profile>
+</profiles>
 </project>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagerPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagerPBClientImpl.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagerPBClientImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagerPBClientImpl.java Fri Aug  3 19:00:15 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.impl.
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.io.Closeable;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -49,7 +50,8 @@ import org.apache.hadoop.yarn.proto.Yarn
 
 import com.google.protobuf.ServiceException;
 
-public class ContainerManagerPBClientImpl implements ContainerManager {
+public class ContainerManagerPBClientImpl implements ContainerManager,
+    Closeable {
 
   // Not a documented config. Only used for tests
   static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Aug  3 19:00:15 2012
@@ -541,15 +541,40 @@ public class YarnConfiguration extends C
   public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
       2000;
 
-  /** Standard Hadoop classes */
+  /**
+   * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
+   * entries
+   */
   public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
       + "application.classpath";
 
+  /**
+   * Default CLASSPATH for YARN applications. A comma-separated list of
+   * CLASSPATH entries
+   */
+  public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
+      "$HADOOP_CONF_DIR", "$HADOOP_COMMON_HOME/share/hadoop/common/*",
+      "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*",
+      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*",
+      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*",
+      "$YARN_HOME/share/hadoop/mapreduce/*",
+      "$YARN_HOME/share/hadoop/mapreduce/lib/*"};
+
   /** Container temp directory */
   public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp";
 
   public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + ".is.minicluster";
 
+  /** Whether to use fixed ports with the minicluster. */
+  public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX
+      + "minicluster.fixed.ports";
+
+  /**
+   * Default is false to be able to run tests concurrently without port
+   * conflicts.
+   */
+  public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+
   public YarnConfiguration() {
     super();
   }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java Fri Aug  3 19:00:15 2012
@@ -92,6 +92,23 @@ public class AggregatedLogFormat {
     }
     
     @Override
+    public int hashCode() {
+      return keyString == null ? 0 : keyString.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj instanceof LogKey) {
+        LogKey other = (LogKey) obj;
+        if (this.keyString == null) {
+          return other.keyString == null;
+        }
+        return this.keyString.equals(other.keyString);
+      }
+      return false;
+    }
+
+    @Override
     public void write(DataOutput out) throws IOException {
       out.writeUTF(this.keyString);
     }
@@ -360,7 +377,33 @@ public class AggregatedLogFormat {
       return valueStream;
     }
 
-    
+    /**
+     * Get a ContainerLogsReader to read the logs for
+     * the specified container.
+     *
+     * @param containerId
+     * @return object to read the container's logs or null if the
+     *         logs could not be found
+     * @throws IOException
+     */
+    public ContainerLogsReader getContainerLogsReader(
+        ContainerId containerId) throws IOException {
+      ContainerLogsReader logReader = null;
+
+      final LogKey containerKey = new LogKey(containerId);
+      LogKey key = new LogKey();
+      DataInputStream valueStream = next(key);
+      while (valueStream != null && !key.equals(containerKey)) {
+        valueStream = next(key);
+      }
+
+      if (valueStream != null) {
+        logReader = new ContainerLogsReader(valueStream);
+      }
+
+      return logReader;
+    }
+
     //TODO  Change Log format and interfaces to be containerId specific.
     // Avoid returning completeValueStreams.
 //    public List<String> getTypesForContainer(DataInputStream valueStream){}
@@ -489,4 +532,67 @@ public class AggregatedLogFormat {
       this.fsDataIStream.close();
     }
   }
+
+  public static class ContainerLogsReader {
+    private DataInputStream valueStream;
+    private String currentLogType = null;
+    private long currentLogLength = 0;
+    private BoundedInputStream currentLogData = null;
+    private InputStreamReader currentLogISR;
+
+    public ContainerLogsReader(DataInputStream stream) {
+      valueStream = stream;
+    }
+
+    public String nextLog() throws IOException {
+      if (currentLogData != null && currentLogLength > 0) {
+        // seek to the end of the current log, relying on BoundedInputStream
+        // to prevent seeking past the end of the current log
+        do {
+          if (currentLogData.skip(currentLogLength) < 0) {
+            break;
+          }
+        } while (currentLogData.read() != -1);
+      }
+
+      currentLogType = null;
+      currentLogLength = 0;
+      currentLogData = null;
+      currentLogISR = null;
+
+      try {
+        String logType = valueStream.readUTF();
+        String logLengthStr = valueStream.readUTF();
+        currentLogLength = Long.parseLong(logLengthStr);
+        currentLogData =
+            new BoundedInputStream(valueStream, currentLogLength);
+        currentLogData.setPropagateClose(false);
+        currentLogISR = new InputStreamReader(currentLogData);
+        currentLogType = logType;
+      } catch (EOFException e) {
+      }
+
+      return currentLogType;
+    }
+
+    public String getCurrentLogType() {
+      return currentLogType;
+    }
+
+    public long getCurrentLogLength() {
+      return currentLogLength;
+    }
+
+    public long skip(long n) throws IOException {
+      return currentLogData.skip(n);
+    }
+
+    public int read(byte[] buf, int off, int len) throws IOException {
+      return currentLogData.read(buf, off, len);
+    }
+
+    public int read(char[] buf, int off, int len) throws IOException {
+      return currentLogISR.read(buf, off, len);
+    }
+  }
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Fri Aug  3 19:00:15 2012
@@ -35,6 +35,11 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 
+/**
+ * TokenIdentifier for a container. Encodes {@link ContainerId},
+ * {@link Resource} needed by the container and the target NMs host-address.
+ * 
+ */
 public class ContainerTokenIdentifier extends TokenIdentifier {
 
   private static Log LOG = LogFactory.getLog(ContainerTokenIdentifier.class);
@@ -44,14 +49,19 @@ public class ContainerTokenIdentifier ex
   private ContainerId containerId;
   private String nmHostAddr;
   private Resource resource;
+  private long expiryTimeStamp;
 
   public ContainerTokenIdentifier(ContainerId containerID, String hostName,
-      Resource r) {
+      Resource r, long expiryTimeStamp) {
     this.containerId = containerID;
     this.nmHostAddr = hostName;
     this.resource = r;
+    this.expiryTimeStamp = expiryTimeStamp;
   }
 
+  /**
+   * Default constructor needed by RPC layer/SecretManager.
+   */
   public ContainerTokenIdentifier() {
   }
 
@@ -67,6 +77,10 @@ public class ContainerTokenIdentifier ex
     return this.resource;
   }
 
+  public long getExpiryTimeStamp() {
+    return this.expiryTimeStamp;
+  }
+
   @Override
   public void write(DataOutput out) throws IOException {
     LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
@@ -79,6 +93,7 @@ public class ContainerTokenIdentifier ex
     out.writeInt(this.containerId.getId());
     out.writeUTF(this.nmHostAddr);
     out.writeInt(this.resource.getMemory());
+    out.writeLong(this.expiryTimeStamp);
   }
 
   @Override
@@ -91,6 +106,7 @@ public class ContainerTokenIdentifier ex
         .readInt());
     this.nmHostAddr = in.readUTF();
     this.resource = BuilderUtils.newResource(in.readInt());
+    this.expiryTimeStamp = in.readLong();
   }
 
   @Override
@@ -103,6 +119,7 @@ public class ContainerTokenIdentifier ex
     return UserGroupInformation.createRemoteUser(this.containerId.toString());
   }
 
+  // TODO: Needed?
   @InterfaceAudience.Private
   public static class Renewer extends Token.TrivialRenewer {
     @Override

Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Fri Aug  3 19:00:15 2012
@@ -331,14 +331,16 @@ public class BuilderUtils {
   }
 
   public static ApplicationReport newApplicationReport(
-      ApplicationId applicationId, String user, String queue, String name,
-      String host, int rpcPort, String clientToken, YarnApplicationState state,
-      String diagnostics, String url, long startTime, long finishTime,
-      FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources,
-      String origTrackingUrl) {
+      ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
+      String user, String queue, String name, String host, int rpcPort,
+      String clientToken, YarnApplicationState state, String diagnostics,
+      String url, long startTime, long finishTime,
+      FinalApplicationStatus finalStatus,
+      ApplicationResourceUsageReport appResources, String origTrackingUrl) {
     ApplicationReport report = recordFactory
         .newRecordInstance(ApplicationReport.class);
     report.setApplicationId(applicationId);
+    report.setCurrentApplicationAttemptId(applicationAttemptId);
     report.setUser(user);
     report.setQueue(queue);
     report.setName(name);



Mime
View raw message