hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1158962 [2/3] - in /hadoop/common/branches/MR-279-merge/hadoop-mapreduce: ./ assembly/ hadoop-mr-client/ hadoop-mr-client/hadoop-mapreduce-client-app/ hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ had...
Date Wed, 17 Aug 2011 22:57:55 GMT

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml Wed Aug 17 22:57:31 2011
@@ -9,7 +9,7 @@
   <!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
   <fileSets>
     <fileSet>
-      <directory>yarn/yarn-server/yarn-server-nodemanager/target/classes/bin</directory>
+      <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin</directory>
       <outputDirectory>bin</outputDirectory>
       <includes>
         <include>container-executor</include>
@@ -17,7 +17,7 @@
       <fileMode>0755</fileMode>
     </fileSet>
     <fileSet>
-      <directory>yarn/bin</directory>
+      <directory>hadoop-yarn/bin</directory>
       <outputDirectory>bin</outputDirectory>
       <includes>
         <include>*</include>
@@ -33,7 +33,7 @@
       <fileMode>0755</fileMode>
     </fileSet>
     <fileSet>
-      <directory>yarn/conf</directory>
+      <directory>hadoop-yarn/conf</directory>
       <outputDirectory>conf</outputDirectory>
       <includes>
         <include>**/*</include>
@@ -62,7 +62,7 @@
         <include>INSTALL</include>
         <include>LICENSE.txt</include>
         <include>mr-client/**</include>
-        <include>yarn/**</include>
+        <include>hadoop-yarn/**</include>
         <include>src/**</include>
       </includes>
     </fileSet>
@@ -78,7 +78,7 @@
   <moduleSets>
     <moduleSet>
       <excludes>
-        <exclude>org.apache.hadoop:yarn-server-tests</exclude>
+        <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
       </excludes>
       <binaries>
         <outputDirectory>modules</outputDirectory>

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml Wed Aug 17 22:57:31 2011
@@ -19,7 +19,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-common</artifactId>
+      <artifactId>hadoop-yarn-common</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
@@ -29,17 +29,17 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-server-common</artifactId>
+      <artifactId>hadoop-yarn-server-common</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-server-nodemanager</artifactId>
+      <artifactId>hadoop-yarn-server-nodemanager</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-server-resourcemanager</artifactId>
+      <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Wed Aug 17 22:57:31 2011
@@ -406,12 +406,6 @@ public class TaskAttemptListenerImpl ext
   }
 
   @Override
-  public void updatePrivateDistributedCacheSizes(
-      org.apache.hadoop.mapreduce.JobID jobId, long[] sizes) throws IOException {
-    // TODO Auto-generated method stub
-  }
-
-  @Override
   public void register(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
       org.apache.hadoop.mapred.Task task, WrappedJvmID jvmID) {
     //create the mapping so that it is easy to look up

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Wed Aug 17 22:57:31 2011
@@ -803,7 +803,7 @@ public class JobImpl implements org.apac
             job.conf.get(MRJobConfig.USER_NAME, "mapred"),
             job.submitTime,
             job.remoteJobConfFile.toString(),
-            job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "test"));
+            job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "default"));
         job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
         //TODO JH Verify jobACLs, UserName via UGI?
 
@@ -1111,7 +1111,6 @@ public class JobImpl implements org.apac
         new JobInitedEvent(job.oldJobId,
              job.startTime,
              job.numMapTasks, job.numReduceTasks,
-             job.isUber, 0, 0,  // FIXME: lose latter two args again (old-style uber junk:  needs to go along with 98% of other old-style uber junk)
              job.getState().toString()); //Will transition to state running. Currently in INITED
       job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
       JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Aug 17 22:57:31 2011
@@ -642,13 +642,24 @@ public abstract class TaskAttemptImpl im
     return container;
   }
 
+  private static long[] parseTimeStamps(String[] strs) {
+    if (null == strs) {
+      return null;
+    }
+    long[] result = new long[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Long.parseLong(strs[i]);
+    }
+    return result;
+  }
+
   private void setupDistributedCache(FileSystem remoteFS, Configuration conf, 
       ContainerLaunchContext container) throws IOException {
     
     // Cache archives
     parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE, 
         DistributedCache.getCacheArchives(conf), 
-        DistributedCache.getArchiveTimestamps(conf), 
+        parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), 
         getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
         DistributedCache.getArchiveVisibilities(conf), 
         DistributedCache.getArchiveClassPaths(conf));
@@ -656,7 +667,7 @@ public abstract class TaskAttemptImpl im
     // Cache files
     parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE, 
         DistributedCache.getCacheFiles(conf),
-        DistributedCache.getFileTimestamps(conf),
+        parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
         getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
         DistributedCache.getFileVisibilities(conf),
         DistributedCache.getFileClassPaths(conf));

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml Wed Aug 17 22:57:31 2011
@@ -18,7 +18,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-common</artifactId>
+      <artifactId>hadoop-yarn-common</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -56,7 +56,7 @@
             <configuration>
               <executable>protoc</executable>
               <arguments>
-                <argument>-I../../yarn/yarn-api/src/main/proto/</argument>
+                <argument>-I../../hadoop-yarn/hadoop-yarn-api/src/main/proto/</argument>
                 <argument>-Isrc/main/proto/</argument>
                 <argument>--java_out=target/generated-sources/proto</argument>
                 <argument>src/main/proto/mr_protos.proto</argument>

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml (from r1158949, hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml Wed Aug 17 22:57:31 2011
@@ -18,7 +18,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>yarn-common</artifactId> 
+      <artifactId>hadoop-yarn-common</artifactId> 
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java Wed Aug 17 22:57:31 2011
@@ -560,7 +560,7 @@ public class BackupStore<K,V> {
 
     private Writer<K,V> createSpillFile() throws IOException {
       Path tmp =
-          new Path(TaskTracker.OUTPUT + "/backup_" + tid.getId() + "_"
+          new Path(Constants.OUTPUT + "/backup_" + tid.getId() + "_"
               + (spillNumber++) + ".out");
 
       LOG.info("Created file: " + tmp);

Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+public class Constants {
+  static final String OUTPUT = "output";
+  public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
+  public static final String JOBFILE = "job.xml";
+  public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
+  public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
+}

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java Wed Aug 17 22:57:31 2011
@@ -48,7 +48,7 @@ public class JobACLsManager {
    * 
    * @return JobACL to AccessControlList map.
    */
-  Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
+  public Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
 
     Map<JobACL, AccessControlList> acls =
         new HashMap<JobACL, AccessControlList>();

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java Wed Aug 17 22:57:31 2011
@@ -54,7 +54,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getOutputFile()
       throws IOException {
-    return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+    return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
         + MAP_OUTPUT_FILENAME_STRING, getConf());
   }
 
@@ -68,7 +68,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getOutputFileForWrite(long size)
       throws IOException {
-    return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+    return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
         + MAP_OUTPUT_FILENAME_STRING, size, getConf());
   }
 
@@ -89,7 +89,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getOutputIndexFile()
       throws IOException {
-    return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+    return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
         + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
         getConf());
   }
@@ -104,7 +104,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getOutputIndexFileForWrite(long size)
       throws IOException {
-    return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+    return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
         + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
         size, getConf());
   }
@@ -128,7 +128,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getSpillFile(int spillNumber)
       throws IOException {
-    return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+    return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
         + spillNumber + ".out", getConf());
   }
 
@@ -143,7 +143,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getSpillFileForWrite(int spillNumber, long size)
       throws IOException {
-    return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+    return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
         + spillNumber + ".out", size, getConf());
   }
 
@@ -157,7 +157,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getSpillIndexFile(int spillNumber)
       throws IOException {
-    return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+    return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
         + spillNumber + ".out.index", getConf());
   }
 
@@ -172,7 +172,7 @@ public class MROutputFiles extends MapOu
   @Override
   public Path getSpillIndexFileForWrite(int spillNumber, long size)
       throws IOException {
-    return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+    return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
         + spillNumber + ".out.index", size, getConf());
   }
 
@@ -187,7 +187,7 @@ public class MROutputFiles extends MapOu
   public Path getInputFile(int mapId)
       throws IOException {
     return lDirAlloc.getLocalPathToRead(String.format(
-        REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer
+        REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, Integer
             .valueOf(mapId)), getConf());
   }
 
@@ -204,7 +204,7 @@ public class MROutputFiles extends MapOu
                                    long size)
       throws IOException {
     return lDirAlloc.getLocalPathForWrite(String.format(
-        REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()),
+        REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, mapId.getId()),
         size, getConf());
   }
 
@@ -212,7 +212,7 @@ public class MROutputFiles extends MapOu
   @Override
   public void removeAll()
       throws IOException {
-    ((JobConf)getConf()).deleteLocalFiles(TaskTracker.OUTPUT);
+    ((JobConf)getConf()).deleteLocalFiles(Constants.OUTPUT);
   }
 
   @Override

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Wed Aug 17 22:57:31 2011
@@ -107,13 +107,6 @@ class MapTask extends Task {
       throws IOException {
     super.localizeConfiguration(conf);
   }
-  
-  
-  @Override
-  public TaskRunner createRunner(TaskTracker tracker, 
-      TaskTracker.TaskInProgress tip) {
-    return new MapTaskRunner(tip, tracker, this.conf);
-  }
 
   @Override
   public void write(DataOutput out) throws IOException {

Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+
+public class Master {
+  
+  public enum State {
+    INITIALIZING, RUNNING;
+  }
+
+  public static String getMasterUserName(Configuration conf) {
+    return conf.get(MRConfig.MASTER_USER_NAME);
+  }
+  
+  public static InetSocketAddress getMasterAddress(Configuration conf) {
+    String jobTrackerStr =
+      conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+    return NetUtils.createSocketAddr(jobTrackerStr);
+  }
+
+  public static String getMasterPrincipal(Configuration conf) 
+  throws IOException {
+    String jtHostname = getMasterAddress(conf).getHostName();
+    // get jobtracker principal for use as delegation token renewer
+    return SecurityUtil.getServerPrincipal(getMasterUserName(conf), jtHostname);
+  }
+  
+}

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java Wed Aug 17 22:57:31 2011
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.TaskScheduler.QueueRefresher;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.security.UserGroupInformation;

Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,49 @@
+package org.apache.hadoop.mapred;
+
+import java.util.List;
+
+/**
+ * Abstract QueueRefresher class. Scheduler's can extend this and return an
+ * instance of this in the {@link #getQueueRefresher()} method. The
+ * {@link #refreshQueues(List)} method of this instance will be invoked by the
+ * {@link QueueManager} whenever it gets a request from an administrator to
+ * refresh its own queue-configuration. This method has a documented contract
+ * between the {@link QueueManager} and the {@link TaskScheduler}.
+ * 
+ * Before calling QueueRefresher, the caller must hold the lock to the
+ * corresponding {@link TaskScheduler} (generally in the {@link JobTracker}).
+ */
+abstract class QueueRefresher {
+
+  /**
+   * Refresh the queue-configuration in the scheduler. This method has the
+   * following contract.
+   * <ol>
+   * <li>Before this method, {@link QueueManager} does a validation of the new
+   * queue-configuration. For e.g, currently addition of new queues, or
+   * removal of queues at any level in the hierarchy is not supported by
+   * {@link QueueManager} and so are not supported for schedulers too.</li>
+   * <li>Schedulers will be passed a list of {@link JobQueueInfo}s of the root
+   * queues i.e. the queues at the top level. All the descendants are properly
+   * linked from these top-level queues.</li>
+   * <li>Schedulers should use the scheduler specific queue properties from
+   * the newRootQueues, validate the properties themselves and apply them
+   * internally.</li>
+   * <li>
+   * Once the method returns successfully from the schedulers, it is assumed
+   * that the refresh of queue properties is successful throughout and will be
+   * 'committed' internally to {@link QueueManager} too. It is guaranteed that
+   * at no point, after successful return from the scheduler, is the queue
+   * refresh in QueueManager failed. If ever, such abnormalities happen, the
+   * queue framework will be inconsistent and will need a JT restart.</li>
+   * <li>If scheduler throws an exception during {@link #refreshQueues()},
+   * {@link QueueManager} throws away the newly read configuration, retains
+   * the old (consistent) configuration and informs the request issuer about
+   * the error appropriately.</li>
+   * </ol>
+   * 
+   * @param newRootQueues
+   */
+  abstract void refreshQueues(List<JobQueueInfo> newRootQueues)
+      throws Throwable;
+}
\ No newline at end of file

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Wed Aug 17 22:57:31 2011
@@ -46,11 +46,10 @@ import org.apache.hadoop.io.SequenceFile
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
-import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.mapreduce.task.reduce.Shuffle;
 import org.apache.hadoop.util.Progress;
 import org.apache.hadoop.util.Progressable;
@@ -145,12 +144,6 @@ public class ReduceTask extends Task {
   }
 
   @Override
-  public TaskRunner createRunner(TaskTracker tracker, TaskInProgress tip) 
-  throws IOException {
-    return new ReduceTaskRunner(tip, tracker, this.conf);
-  }
-
-  @Override
   public boolean isMapTask() {
     return false;
   }
@@ -347,7 +340,7 @@ public class ReduceTask extends Task {
     // Initialize the codec
     codec = initCodec();
     RawKeyValueIterator rIter = null;
-    boolean isLocal = "local".equals(job.get(JTConfig.JT_IPC_ADDRESS, "local"));
+    boolean isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
     if (!isLocal) {
       Class combinerClass = conf.getCombinerClass();
       CombineOutputCollector combineCollector = 

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java Wed Aug 17 22:57:31 2011
@@ -34,7 +34,7 @@ import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SecureIOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.PureJavaCrc32;
-
+import org.apache.hadoop.fs.FSDataInputStream;
 import static org.apache.hadoop.mapred.MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH;
 
 class SpillRecord {
@@ -64,9 +64,7 @@ class SpillRecord {
       throws IOException {
 
     final FileSystem rfs = FileSystem.getLocal(job).getRaw();
-    final DataInputStream in = 
-      new DataInputStream(SecureIOUtils.openForRead(
-          new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
+    final FSDataInputStream in = rfs.open(indexFileName);
     try {
       final long length = rfs.getFileStatus(indexFileName).getLen();
       final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Wed Aug 17 22:57:31 2011
@@ -61,9 +61,8 @@ import org.apache.hadoop.mapreduce.MRCon
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
 import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin.*;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.*;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.Progress;
 import org.apache.hadoop.util.Progressable;
@@ -487,12 +486,6 @@ abstract public class Task implements Wr
   public abstract void run(JobConf job, TaskUmbilicalProtocol umbilical)
     throws IOException, ClassNotFoundException, InterruptedException;
 
-
-  /** Return an approprate thread runner for this task. 
-   * @param tip TODO*/
-  public abstract TaskRunner createRunner(TaskTracker tracker, 
-      TaskTracker.TaskInProgress tip) throws IOException;
-
   /** The number of milliseconds between progress reports. */
   public static final int PROGRESS_INTERVAL = 3000;
 
@@ -539,7 +532,7 @@ abstract public class Task implements Wr
     }
     committer.setupTask(taskContext);
     Class<? extends ResourceCalculatorPlugin> clazz =
-        conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
+        conf.getClass(MRConfig.RESOURCE_CALCULATOR_PLUGIN,
             null, ResourceCalculatorPlugin.class);
     resourceCalculator = ResourceCalculatorPlugin
             .getResourceCalculatorPlugin(clazz, conf);
@@ -1161,7 +1154,7 @@ abstract public class Task implements Wr
     // add the static resolutions (this is required for the junit to
     // work on testcases that simulate multiple nodes on a single physical
     // node.
-    String hostToResolved[] = conf.getStrings(TTConfig.TT_STATIC_RESOLUTIONS);
+    String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
     if (hostToResolved != null) {
       for (String str : hostToResolved) {
         String name = str.substring(0, str.indexOf('='));
@@ -1175,6 +1168,10 @@ abstract public class Task implements Wr
     return this.conf;
   }
 
+  public MapOutputFile getMapOutputFile() {
+    return mapOutputFile;
+  }
+
   /**
    * OutputCollector for the combiner.
    */

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java Wed Aug 17 22:57:31 2011
@@ -33,12 +33,10 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
+import org.apache.hadoop.mapred.Master;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
 import org.apache.hadoop.mapreduce.util.ConfigUtil;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
@@ -281,8 +279,8 @@ public class Cluster {
     if (jobHistoryDir == null) {
       jobHistoryDir = new Path(client.getJobHistoryDir());
     }
-    return JobHistory.getJobHistoryFile(jobHistoryDir, jobId, 
-        ugi.getShortUserName()).toString();
+    return new Path(jobHistoryDir, jobId.toString() + "_"
+                    + ugi.getShortUserName()).toString();
   }
 
   /**
@@ -314,19 +312,6 @@ public class Cluster {
       throws IOException, InterruptedException {
     return client.getChildQueues(queueName);
   }
-
-  /**
-   * Get JobTracker's state
-   * 
-   * @return {@link State} of the JobTracker
-   * @throws IOException
-   * @throws InterruptedException
-   * @deprecated Use {@link #getJobTrackerStatus()} instead.
-   */
-  @Deprecated
-  public State getJobTrackerState() throws IOException, InterruptedException {
-    return client.getJobTrackerState();
-  }
   
   /**
    * Get the JobTracker's status.
@@ -359,8 +344,7 @@ public class Cluster {
       getDelegationToken(Text renewer) throws IOException, InterruptedException{
     Token<DelegationTokenIdentifier> result =
       client.getDelegationToken(renewer);
-    InetSocketAddress addr = NetUtils.createSocketAddr(
-        conf.get(JTConfig.JT_IPC_ADDRESS, "localhost:8012"));
+    InetSocketAddress addr = Master.getMasterAddress(conf);
     StringBuilder service = new StringBuilder();
     service.append(NetUtils.normalizeHostName(addr.getAddress().
                                               getHostAddress()));

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java Wed Aug 17 22:57:31 2011
@@ -35,5 +35,9 @@ public enum JobCounter {
   SLOTS_MILLIS_MAPS,
   SLOTS_MILLIS_REDUCES,
   FALLOW_SLOTS_MILLIS_MAPS,
-  FALLOW_SLOTS_MILLIS_REDUCES
+  FALLOW_SLOTS_MILLIS_REDUCES,
+  TOTAL_LAUNCHED_UBERTASKS,
+  NUM_UBER_SUBMAPS,
+  NUM_UBER_SUBREDUCES,
+  NUM_FAILED_UBERTASKS
 }

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Wed Aug 17 22:57:31 2011
@@ -18,8 +18,6 @@
 package org.apache.hadoop.mapreduce;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapred.TaskTracker;
 
 /**
  * Place holder for cluster level configuration keys.
@@ -58,6 +56,15 @@ public interface MRConfig {
   public static final long    DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 
     7*24*60*60*1000; // 7 days
   
+  public static final String RESOURCE_CALCULATOR_PLUGIN = 
+    "mapreduce.job.resourcecalculatorplugin";
+  public static final String STATIC_RESOLUTIONS = 
+    "mapreduce.job.net.static.resolutions";
+
+  public static final String MASTER_ADDRESS  = "mapreduce.jobtracker.address";
+  public static final String MASTER_USER_NAME = 
+    "mapreduce.jobtracker.kerberos.principal";
+
   public static final String FRAMEWORK_NAME  = "mapreduce.framework.name";
   public static final String TASK_LOCAL_OUTPUT_CLASS =
   "mapreduce.task.local.output.class";

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Aug 17 22:57:31 2011
@@ -287,4 +287,18 @@ public interface MRJobConfig {
 
   public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max";
   public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
+  public static final String JOB_UBERTASK_ENABLE =
+    "mapreduce.job.ubertask.enable";
+  public static final String JOB_UBERTASK_MAXMAPS =
+    "mapreduce.job.ubertask.maxmaps";
+  public static final String JOB_UBERTASK_MAXREDUCES =
+    "mapreduce.job.ubertask.maxreduces";
+  public static final String JOB_UBERTASK_MAXBYTES =
+    "mapreduce.job.ubertask.maxbytes";
+  public static final String UBERTASK_JAVA_OPTS =
+    "mapreduce.ubertask.child.java.opts";  // or mapreduce.uber.java.opts?
+  public static final String UBERTASK_ULIMIT =
+    "mapreduce.ubertask.child.ulimit";     // or mapreduce.uber.ulimit?
+  public static final String UBERTASK_ENV =
+    "mapreduce.ubertask.child.env";        // or mapreduce.uber.env?
 }

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java Wed Aug 17 22:57:31 2011
@@ -377,6 +377,38 @@ public class DistributedCache {
     return false;
   }
 
+  private static boolean[] parseBooleans(String[] strs) {
+    if (null == strs) {
+      return null;
+    }
+    boolean[] result = new boolean[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Boolean.parseBoolean(strs[i]);
+    }
+    return result;
+  }
+
+  /**
+   * Get the booleans on whether the files are public or not.  Used by 
+   * internal DistributedCache and MapReduce code.
+   * @param conf The configuration which stored the timestamps
+   * @return a string array of booleans 
+   * @throws IOException
+   */
+  public static boolean[] getFileVisibilities(Configuration conf) {
+    return parseBooleans(conf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES));
+  }
+
+  /**
+   * Get the booleans on whether the archives are public or not.  Used by 
+   * internal DistributedCache and MapReduce code.
+   * @param conf The configuration which stored the timestamps
+   * @return a string array of booleans 
+   */
+  public static boolean[] getArchiveVisibilities(Configuration conf) {
+    return parseBooleans(conf.getStrings(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES));
+  }
+
   /**
    * This method checks if there is a conflict in the fragment names 
    * of the uris. Also makes sure that each uri has a fragment. It 

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java Wed Aug 17 22:57:31 2011
@@ -18,12 +18,11 @@
 
 package org.apache.hadoop.mapreduce.jobhistory;
 
-import java.lang.Integer;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 import org.apache.avro.Schema;
-
-import org.apache.avro.generic.GenericArray;
 import org.apache.avro.generic.GenericData;
 
 public class AvroArrayUtils {
@@ -31,13 +30,12 @@ public class AvroArrayUtils {
   private static final Schema ARRAY_INT
       = Schema.createArray(Schema.create(Schema.Type.INT));
 
-  static public GenericArray<Integer> NULL_PROGRESS_SPLITS_ARRAY
+  static public List<Integer> NULL_PROGRESS_SPLITS_ARRAY
     = new GenericData.Array<Integer>(0, ARRAY_INT);
 
-  public static GenericArray<Integer>
+  public static List<Integer>
     toAvro(int values[]) {
-    GenericData.Array<Integer> result
-      = new GenericData.Array<Integer>(values.length, ARRAY_INT);
+    List<Integer> result = new ArrayList<Integer>(values.length);
 
     for (int i = 0; i < values.length; ++i) {
       result.add(values[i]);
@@ -46,7 +44,7 @@ public class AvroArrayUtils {
     return result;
   }
 
-  public static int[] fromAvro(GenericArray<Integer> avro) {
+  public static int[] fromAvro(List<Integer> avro) {
     int[] result = new int[(int)avro.size()];
 
     int i = 0;

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Wed Aug 17 22:57:31 2011
@@ -19,23 +19,21 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
+import java.util.ArrayList;
 
 import org.apache.avro.Schema;
+import org.apache.avro.io.DatumWriter;
 import org.apache.avro.io.Encoder;
 import org.apache.avro.io.JsonEncoder;
-import org.apache.avro.io.DatumWriter;
 import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
 import org.apache.avro.util.Utf8;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
 
 /**
  * Event Writer is an utility class used to write events to the underlying
@@ -97,13 +95,13 @@ class EventWriter {
   static JhCounters toAvro(Counters counters, String name) {
     JhCounters result = new JhCounters();
     result.name = new Utf8(name);
-    result.groups = new GenericData.Array<JhCounterGroup>(0, GROUPS);
+    result.groups = new ArrayList<JhCounterGroup>(0);
     if (counters == null) return result;
     for (CounterGroup group : counters) {
       JhCounterGroup g = new JhCounterGroup();
       g.name = new Utf8(group.getName());
       g.displayName = new Utf8(group.getDisplayName());
-      g.counts = new GenericData.Array<JhCounter>(group.size(), COUNTERS);
+      g.counts = new ArrayList<JhCounter>(group.size());
       for (Counter counter : group) {
         JhCounter c = new JhCounter();
         c.name = new Utf8(counter.getName());

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java Wed Aug 17 22:57:31 2011
@@ -34,9 +34,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobInProgress;
 import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.TaskLogServlet;
 import org.apache.hadoop.mapred.TaskStatus;
 import org.apache.hadoop.mapreduce.CounterGroup;
 import org.apache.hadoop.mapreduce.Counters;
@@ -45,6 +43,7 @@ import org.apache.hadoop.mapreduce.TaskI
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.util.HostUtil;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -456,9 +455,9 @@ public class HistoryViewer {
     }
   
     String taskTrackerName =
-      JobInProgress.convertTrackerNameToHostName(
+      HostUtil.convertTrackerNameToHostName(
         attempt.getTrackerName());
-    return TaskLogServlet.getTaskLogUrl(taskTrackerName,
+    return HostUtil.getTaskLogUrl(taskTrackerName,
         Integer.toString(attempt.getHttpPort()),
         attempt.getAttemptId().toString());
   }

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Wed Aug 17 22:57:31 2011
@@ -57,7 +57,7 @@ public class JobSubmittedEvent implement
     datum.userName = new Utf8(userName);
     datum.submitTime = submitTime;
     datum.jobConfPath = new Utf8(jobConfPath);
-    Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+    Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
     for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
       jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
           entry.getValue().getAclString()));

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Wed Aug 17 22:57:31 2011
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.TaskT
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.authorize.AccessControlList;
@@ -140,17 +139,6 @@ public interface ClientProtocol extends 
    */
   public ClusterMetrics getClusterMetrics() 
   throws IOException, InterruptedException;
-  
-  /**
-   * Get JobTracker's state
-   * 
-   * @return {@link State} of the JobTracker
-   * @throws IOException
-   * @throws InterruptedException
-   * @deprecated Use {@link #getJobTrackerStatus()} instead.
-   */
-  @Deprecated
-  public State getJobTrackerState() throws IOException, InterruptedException;
 
   /**
    * Get the JobTracker's status.

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Wed Aug 17 22:57:31 2011
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.Master;
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.security.Credentials;
@@ -91,13 +91,6 @@ public class TokenCache {
     }
   }
   
-  static String getJTPrincipal(Configuration conf) throws IOException {
-    String jtHostname = JobTracker.getAddress(conf).getHostName();
-    // get jobtracker principal for use as delegation token renewer
-    return SecurityUtil.getServerPrincipal(conf.get(JTConfig.JT_USER_NAME),
-        jtHostname);
-  }
-  
   /**
    * get delegation token for a specific FS
    * @param fs
@@ -108,7 +101,7 @@ public class TokenCache {
    */
   static void obtainTokensForNamenodesInternal(FileSystem fs, 
       Credentials credentials, Configuration conf) throws IOException {
-    String delegTokenRenewer = getJTPrincipal(conf);
+    String delegTokenRenewer = Master.getMasterPrincipal(conf);
     if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
       throw new IOException(
           "Can't get JobTracker Kerberos principal for use as renewer");

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java Wed Aug 17 22:57:31 2011
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
-import org.apache.hadoop.mapred.TaskTracker;
 import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java Wed Aug 17 22:57:31 2011
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.LocalDirAllo
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BoundedByteArrayOutputStream;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskTracker;
+import org.apache.hadoop.mapred.MapOutputFile;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 
 class MapOutput<K,V> {
@@ -64,7 +64,8 @@ class MapOutput<K,V> {
   
   MapOutput(TaskAttemptID mapId, MergeManager<K,V> merger, long size, 
             JobConf conf, LocalDirAllocator localDirAllocator,
-            int fetcher, boolean primaryMapOutput)  throws IOException {
+            int fetcher, boolean primaryMapOutput, MapOutputFile mapOutputFile)
+         throws IOException {
     this.id = ID.incrementAndGet();
     this.mapId = mapId;
     this.merger = merger;
@@ -77,18 +78,10 @@ class MapOutput<K,V> {
     this.size = size;
     
     this.localFS = FileSystem.getLocal(conf);
-    String filename = "map_" + mapId.getTaskID().getId() + ".out";
-    String tmpOutput = Path.SEPARATOR +
-                            TaskTracker.getJobCacheSubdir(conf.getUser()) +
-                       Path.SEPARATOR + mapId.getJobID() +
-                       Path.SEPARATOR + merger.getReduceId() +
-                       Path.SEPARATOR + "output" + 
-                       Path.SEPARATOR + filename + 
-                       "." + fetcher; 
-
-    tmpOutputPath = 
-      localDirAllocator.getLocalPathForWrite(tmpOutput, size, conf);
-    outputPath = new Path(tmpOutputPath.getParent(), filename);
+    outputPath =
+      mapOutputFile.getInputFileForWrite(mapId.getTaskID(),size);
+    tmpOutputPath = outputPath.suffix(String.valueOf(fetcher));
+
     disk = localFS.create(tmpOutputPath);
     
     this.primaryMapOutput = primaryMapOutput;

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Wed Aug 17 22:57:31 2011
@@ -225,7 +225,8 @@ public class MergeManager<K, V> {
                " is greater than maxSingleShuffleLimit (" + 
                maxSingleShuffleLimit + ")");
       return new MapOutput<K,V>(mapId, this, requestedSize, jobConf, 
-                                localDirAllocator, fetcher, true);
+                                localDirAllocator, fetcher, true,
+                                mapOutputFile);
     }
     
     // Stall shuffle if we are above the memory limit

Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,33 @@
+package org.apache.hadoop.mapreduce.util;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public class HostUtil {
+
+  /**
+   * Construct the taskLogUrl
+   * @param taskTrackerHostName
+   * @param httpPort
+   * @param taskAttemptID
+   * @return the taskLogUrl
+   */
+  public static String getTaskLogUrl(String taskTrackerHostName,
+      String httpPort, String taskAttemptID) {
+    return ("http://" + taskTrackerHostName + ":" + httpPort
+        + "/tasklog?attemptid=" + taskAttemptID);
+  }
+
+  public static String convertTrackerNameToHostName(String trackerName) {
+    // Ugly!
+    // Convert the trackerName to its host name
+    int indexOfColon = trackerName.indexOf(":");
+    String trackerHostName = (indexOfColon == -1) ? 
+      trackerName : 
+      trackerName.substring(0, indexOfColon);
+    return trackerHostName.substring("tracker_".length());
+  }
+
+}

Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java Wed Aug 17 22:57:31 2011
@@ -29,7 +29,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.mapred.TaskTrackerStatus;
 
 /**
  * Plugin to calculate resource information on Linux systems.
@@ -40,6 +39,8 @@ public class LinuxResourceCalculatorPlug
   private static final Log LOG =
       LogFactory.getLog(LinuxResourceCalculatorPlugin.class);
 
+  public static final int UNAVAILABLE = -1;
+
   /**
    * proc's meminfo virtual file has keys-values in the format
    * "key:[ \t]*value[ \t]kB".
@@ -87,9 +88,9 @@ public class LinuxResourceCalculatorPlug
   private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms)
   private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms)
   // Unix timestamp while reading the CPU time (ms)
-  private float cpuUsage = TaskTrackerStatus.UNAVAILABLE;
-  private long sampleTime = TaskTrackerStatus.UNAVAILABLE;
-  private long lastSampleTime = TaskTrackerStatus.UNAVAILABLE;
+  private float cpuUsage = UNAVAILABLE;
+  private long sampleTime = UNAVAILABLE;
+  private long lastSampleTime = UNAVAILABLE;
   private ProcfsBasedProcessTree pTree = null;
 
   boolean readMemInfoFile = false;
@@ -352,7 +353,7 @@ public class LinuxResourceCalculatorPlug
   public float getCpuUsage() {
     readProcStatFile();
     sampleTime = getCurrentTime();
-    if (lastSampleTime == TaskTrackerStatus.UNAVAILABLE ||
+    if (lastSampleTime == UNAVAILABLE ||
         lastSampleTime > sampleTime) {
       // lastSampleTime > sampleTime may happen when the system time is changed
       lastSampleTime = sampleTime;

Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,236 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.util;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin;
+import org.junit.Test;
+
+/**
+ * A JUnit test to test {@link LinuxResourceCalculatorPlugin}
+ * Create the fake /proc/ information and verify the parsing and calculation
+ */
+public class TestLinuxResourceCalculatorPlugin extends TestCase {
+  /**
+   * LinuxResourceCalculatorPlugin with a fake timer
+   */
+  static class FakeLinuxResourceCalculatorPlugin extends
+      LinuxResourceCalculatorPlugin {
+    
+	  long currentTime = 0;
+	  public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
+			                                       String procfsCpuFile,
+			                                       String procfsStatFile,
+			                                       long jiffyLengthInMillis) {
+	    super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis);
+	  }
+	  @Override
+	  long getCurrentTime() {
+	    return currentTime;
+	  }
+	  public void advanceTime(long adv) {
+	    currentTime += adv * jiffyLengthInMillis;
+	  }
+  }
+  private static final FakeLinuxResourceCalculatorPlugin plugin;
+  private static String TEST_ROOT_DIR = new Path(System.getProperty(
+         "test.build.data", "/tmp")).toString().replace(' ', '+');
+  private static final String FAKE_MEMFILE;
+  private static final String FAKE_CPUFILE;
+  private static final String FAKE_STATFILE;
+  private static final long FAKE_JIFFY_LENGTH = 10L;
+  static {
+    int randomNum = (new Random()).nextInt(1000000000);
+    FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
+    FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
+    FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
+    plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
+                                                   FAKE_STATFILE,
+                                                   FAKE_JIFFY_LENGTH);
+  }
+  static final String MEMINFO_FORMAT = 
+	  "MemTotal:      %d kB\n" +
+	  "MemFree:         %d kB\n" +
+	  "Buffers:        138244 kB\n" +
+	  "Cached:         947780 kB\n" +
+	  "SwapCached:     142880 kB\n" +
+	  "Active:        3229888 kB\n" +
+	  "Inactive:       %d kB\n" +
+	  "SwapTotal:     %d kB\n" +
+	  "SwapFree:      %d kB\n" +
+	  "Dirty:          122012 kB\n" +
+	  "Writeback:           0 kB\n" +
+	  "AnonPages:     2710792 kB\n" +
+	  "Mapped:          24740 kB\n" +
+	  "Slab:           132528 kB\n" +
+	  "SReclaimable:   105096 kB\n" +
+	  "SUnreclaim:      27432 kB\n" +
+	  "PageTables:      11448 kB\n" +
+	  "NFS_Unstable:        0 kB\n" +
+	  "Bounce:              0 kB\n" +
+	  "CommitLimit:   4125904 kB\n" +
+	  "Committed_AS:  4143556 kB\n" +
+	  "VmallocTotal: 34359738367 kB\n" +
+	  "VmallocUsed:      1632 kB\n" +
+	  "VmallocChunk: 34359736375 kB\n" +
+	  "HugePages_Total:     0\n" +
+	  "HugePages_Free:      0\n" +
+	  "HugePages_Rsvd:      0\n" +
+	  "Hugepagesize:     2048 kB";
+  
+  static final String CPUINFO_FORMAT =
+    "processor : %s\n" +
+    "vendor_id : AuthenticAMD\n" +
+    "cpu family  : 15\n" +
+    "model   : 33\n" +
+    "model name  : Dual Core AMD Opteron(tm) Processor 280\n" +
+    "stepping  : 2\n" +
+    "cpu MHz   : %f\n" +
+    "cache size  : 1024 KB\n" +
+    "physical id : 0\n" +
+    "siblings  : 2\n" +
+    "core id   : 0\n" +
+    "cpu cores : 2\n" +
+    "fpu   : yes\n" +
+    "fpu_exception : yes\n" +
+    "cpuid level : 1\n" +
+    "wp    : yes\n" +
+    "flags   : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " +
+    "pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " +
+    "3dnowext 3dnow pni lahf_lm cmp_legacy\n" +
+    "bogomips  : 4792.41\n" +
+    "TLB size  : 1024 4K pages\n" +
+    "clflush size  : 64\n" +
+    "cache_alignment : 64\n" +
+    "address sizes : 40 bits physical, 48 bits virtual\n" +
+    "power management: ts fid vid ttp";
+  
+  static final String STAT_FILE_FORMAT = 
+    "cpu  %d %d %d 1646495089 831319 48713 164346 0\n" +
+    "cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" +
+    "cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" +
+    "cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" +
+    "cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" +
+    "intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" +
+    "ctxt 242017731764\n" +
+    "btime 1257808753\n" +
+    "processes 26414943\n" +
+    "procs_running 1\n" +
+    "procs_blocked 0\n";
+  
+  /**
+   * Test parsing /proc/stat and /proc/cpuinfo
+   * @throws IOException
+   */
+  @Test
+  public void testParsingProcStatAndCpuFile() throws IOException {
+    // Write fake /proc/cpuinfo file.
+    long numProcessors = 8;
+    long cpuFrequencyKHz = 2392781;
+    String fileContent = "";
+    for (int i = 0; i < numProcessors; i++) {
+      fileContent += String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D) +
+                     "\n";
+    }
+    File tempFile = new File(FAKE_CPUFILE);
+    tempFile.deleteOnExit();
+    FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
+    fWriter.write(fileContent);
+    fWriter.close();
+    assertEquals(plugin.getNumProcessors(), numProcessors);
+    assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz);
+    
+    // Write fake /proc/stat file.
+    long uTime = 54972994;
+    long nTime = 188860;
+    long sTime = 19803373;
+    tempFile = new File(FAKE_STATFILE);
+    tempFile.deleteOnExit();
+    updateStatFile(uTime, nTime, sTime);
+    assertEquals(plugin.getCumulativeCpuTime(),
+                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+    assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE));
+    
+    // Advance the time and sample again to test the CPU usage calculation
+    uTime += 100L;
+    plugin.advanceTime(200L);
+    updateStatFile(uTime, nTime, sTime);
+    assertEquals(plugin.getCumulativeCpuTime(),
+                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+    assertEquals(plugin.getCpuUsage(), 6.25F);
+    
+    // Advance the time and sample again. This time, we call getCpuUsage() only.
+    uTime += 600L;
+    plugin.advanceTime(300L);
+    updateStatFile(uTime, nTime, sTime);
+    assertEquals(plugin.getCpuUsage(), 25F);
+    
+    // Advance very short period of time (one jiffy length).
+    // In this case, CPU usage should not be updated.
+    uTime += 1L;
+    plugin.advanceTime(1L);
+    updateStatFile(uTime, nTime, sTime);
+    assertEquals(plugin.getCumulativeCpuTime(),
+                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+    assertEquals(plugin.getCpuUsage(), 25F); // CPU usage is not updated.
+  }
+  
+  /**
+   * Write information to fake /proc/stat file
+   */
+  private void updateStatFile(long uTime, long nTime, long sTime)
+    throws IOException {
+    FileWriter fWriter = new FileWriter(FAKE_STATFILE);
+    fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime));
+    fWriter.close();
+  }
+  
+  /**
+   * Test parsing /proc/meminfo
+   * @throws IOException
+   */
+  @Test
+  public void testParsingProcMemFile() throws IOException {
+    long memTotal = 4058864L;
+    long memFree = 99632L;
+    long inactive = 567732L;
+    long swapTotal = 2096472L;
+    long swapFree = 1818480L;
+    File tempFile = new File(FAKE_MEMFILE);
+    tempFile.deleteOnExit();
+    FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+    fWriter.write(String.format(MEMINFO_FORMAT,
+      memTotal, memFree, inactive, swapTotal, swapFree));
+    
+    fWriter.close();
+    assertEquals(plugin.getAvailablePhysicalMemorySize(),
+                 1024L * (memFree + inactive));
+    assertEquals(plugin.getAvailableVirtualMemorySize(),
+                 1024L * (memFree + inactive + swapFree));
+    assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
+    assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
+  }
+}

Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Wed Aug 17 22:57:31 2011
@@ -94,12 +94,12 @@ public class CompletedJob implements org
 
   @Override
   public int getCompletedMaps() {
-    return jobInfo.getFinishedMaps();
+    return (int) jobInfo.getFinishedMaps();
   }
 
   @Override
   public int getCompletedReduces() {
-    return jobInfo.getFinishedReduces();
+    return (int) jobInfo.getFinishedReduces();
   }
 
   @Override
@@ -264,17 +264,17 @@ public class CompletedJob implements org
 
   @Override
   public int getTotalMaps() {
-    return jobInfo.getTotalMaps();
+    return (int) jobInfo.getTotalMaps();
   }
 
   @Override
   public int getTotalReduces() {
-    return jobInfo.getTotalReduces();
+    return (int) jobInfo.getTotalReduces();
   }
 
   @Override
   public boolean isUber() {
-    return jobInfo.getIsUber();
+    throw new YarnException("Not yet implemented!");
   }
 
   @Override



Mime
View raw message