hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1196458 [18/19] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ assembly/ bin/ conf/ dev-support/ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoop-mapreduce-cl...
Date Wed, 02 Nov 2011 05:35:03 GMT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm Wed Nov  2 05:34:31 2011
@@ -21,7 +21,7 @@ Hadoop MapReduce Next Generation - Writi
 
   \[ {{{./index.html}Go Back}} \]
 
-%{toc|section=1|fromDepth=1}
+%{toc|section=1|fromDepth=0}
 
 * Purpose 
 
@@ -323,25 +323,29 @@ Hadoop MapReduce Next Generation - Writi
     multi-tenancy nature, amongst other issues, it cannot make any assumptions 
     of things like pre-configured ports that it can listen on. 
   
+  * When the ApplicationMaster starts up, several parameters are made available
+    to it via the environment. These include the ContainerId for the
+    ApplicationMaster container, the application submission time and details
+    about the NodeManager host running the Application Master.
+    Ref ApplicationConstants for parameter names.
+
   * All interactions with the ResourceManager require an ApplicationAttemptId 
-    (there can be multiple attempts per application in case of failures). When 
-    the ApplicationMaster starts up, the ApplicationAttemptId associated with 
-    this particular instance will be set in the environment. There are helper 
-    apis to convert the value obtained from the environment into an 
-    ApplicationAttemptId object.
+    (there can be multiple attempts per application in case of failures). The 
+    ApplicationAttemptId can be obtained from the ApplicationMaster
+    containerId. There are helper apis to convert the value obtained from the
+    environment into objects.
     
 +---+
     Map<String, String> envs = System.getenv();
-    ApplicationAttemptId appAttemptID = 
-        Records.newRecord(ApplicationAttemptId.class);
-    if (!envs.containsKey(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV)) {
-      // app attempt id should always be set in the env by the framework 
+    String containerIdString = 
+        envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+    if (containerIdString == null) {
+      // container id should always be set in the env by the framework 
       throw new IllegalArgumentException(
-          "ApplicationAttemptId not set in the environment");				
-    } 
-    appAttemptID = 
-        ConverterUtils.toApplicationAttemptId(
-            envs.get(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV));
+          "ContainerId not set in the environment");
+    }
+    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+    ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
 +---+     
   
   * After an ApplicationMaster has initialized itself completely, it needs to 
@@ -402,7 +406,8 @@ Hadoop MapReduce Next Generation - Writi
     * Resource capability: Currently, YARN only supports memory based resource 
       requirements so the request should define how much memory is needed. The 
       value is defined in MB and has to less than the max capability of the 
-      cluster and an exact multiple of the min capability. 
+      cluster and an exact multiple of the min capability. Memory resources
+      correspond to physical memory limits imposed on the task containers.
       
     * Priority: When asking for sets of containers, an ApplicationMaster may 
       define different priorities to each set. For example, the Map-Reduce 
@@ -770,8 +775,9 @@ Hadoop MapReduce Next Generation - Writi
   The two things you're interested in are physical memory and virtual memory. 
   If you have exceeded physical memory limits your app is using too much physical 
   memory. If you're running a Java app, you can use -hprof to look at what is 
-  taking up space in the heap. If you have exceeded virtual memory, things are 
-  slightly more complicated. 
+  taking up space in the heap. If you have exceeded virtual memory, you may
+  need to increase the value of the the cluster-wide configuration variable
+  <<<yarn.nodemanager.vmem-pmem-ratio>>>.
 
 * Useful Links
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Wed Nov  2 05:34:31 2011
@@ -11,14 +11,12 @@
 ~~ limitations under the License. See accompanying LICENSE file.
 
   ---
-  Hadoop MapReduce Next Generation  ${project.version}
+  Apache Hadoop NextGen MapReduce
   ---
   ---
   ${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation
-
-* Architecture
+  
+MapReduce NextGen aka YARN aka MRv2
 
   The new architecture introduced in hadoop-0.23, divides the two major 
   functions of the JobTracker: resource management and job life-cycle management 
@@ -32,18 +30,21 @@ Hadoop MapReduce Next Generation
   or a DAG of such jobs. 
   
   The ResourceManager and per-machine NodeManager daemon, which manages the 
-  user processes on that machine, form the computation fabric. The 
-  per-application ApplicationMaster is, in effect, a framework specific library 
-  and is tasked with negotiating resources from the ResourceManager and working 
-  with the NodeManager(s) to execute and monitor the tasks.
-
-* User Documentation
+  user processes on that machine, form the computation fabric. 
+  
+  The per-application ApplicationMaster is, in effect, a framework specific 
+  library and is tasked with negotiating resources from the ResourceManager and 
+  working with the NodeManager(s) to execute and monitor the tasks.
 
-  * {{{./SingleCluster.html}Setting up a Single Node Cluster}}
+  More details are available in the {{{./YARN.html}YARN}}
+  document.
 
-  * {{{./apidocs/index.html}JavaDocs}}
+* Documentation
 
+  * {{{./YARN.html}NextGen MapReduce}}
+  
   * {{{./WritingYarnApplications.html}Writing Yarn Applications}}
 
   * {{{./CapacityScheduler.html}Capacity Scheduler}}
 
+

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml Wed Nov  2 05:34:31 2011
@@ -11,18 +11,12 @@
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
 -->
-<project name="Hadoop MapReduce Next Gen">
-
-  <version position="right"/>
-
-  <bannerLeft>
-    <name>&nbsp;</name>
-  </bannerLeft>
+<project name="Apache Hadoop 0.23">
 
   <skin>
     <groupId>org.apache.maven.skins</groupId>
     <artifactId>maven-stylus-skin</artifactId>
-    <version>1.1</version>
+    <version>1.2</version>
   </skin>
 
   <body>

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml Wed Nov  2 05:34:31 2011
@@ -14,24 +14,37 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>0.24.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-yarn</artifactId>
-  <version>${yarn.version}</version>
+  <version>0.24.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>hadoop-yarn</name>
-  <url>http://hadoop.apache.org/mapreduce</url>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <test.logs>true</test.logs>
     <test.timeout>600000</test.timeout>
-    <hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
-    <hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
-    <yarn.version>0.24.0-SNAPSHOT</yarn.version>
-    <install.pom>${project.build.directory}/saner-pom.xml</install.pom>
-    <install.file>${install.pom}</install.file>
     <yarn.basedir>${basedir}</yarn.basedir>
   </properties>
+  
+  <distributionManagement>
+    <repository>
+      <id>apache.releases.https</id>
+      <name>Apache Release Distribution Repository</name>
+      <url>https://repository.apache.org/service/local/staging/deploy/maven2</url>
+    </repository>
+    <snapshotRepository>
+      <id>apache.snapshots.https</id>
+      <name>Apache Development Snapshot Repository</name>
+      <url>https://repository.apache.org/content/repositories/snapshots</url>
+    </snapshotRepository>
+  </distributionManagement>
 
   <repositories>
     <repository>
@@ -54,6 +67,126 @@
 
   <dependencies>
     <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>2.4.0a</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+      <version>1.5.3</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.ant</groupId>
+          <artifactId>ant</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jboss.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.velocity</groupId>
+          <artifactId>velocity</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <artifactId>paranamer-ant</artifactId>
+          <groupId>com.thoughtworks.paranamer</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-el</groupId>
+          <artifactId>commons-el</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-compiler</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jsp-2.1-jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>hsqldb</groupId>
+          <artifactId>hsqldb</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+     <groupId>org.slf4j</groupId>
+       <artifactId>slf4j-api</artifactId>
+       <version>1.6.1</version>
+    </dependency>
+    <dependency>
+     <groupId>org.slf4j</groupId>
+       <artifactId>slf4j-log4j12</artifactId>
+       <version>1.6.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.8.5</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <!-- needed for security and runtime -->
+      <artifactId>hadoop-hdfs</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject.extensions</groupId>
+      <artifactId>guice-servlet</artifactId>
+      <version>2.0</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.8.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.jboss.netty</groupId>
+      <artifactId>netty</artifactId>
+      <version>3.2.3.Final</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cenqua.clover</groupId>
+      <artifactId>clover</artifactId>
+      <version>3.0.2</version>
+    </dependency>
+
+    <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
       <version>1.5.3</version>
@@ -92,7 +225,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.avro</groupId>
@@ -123,7 +256,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-annotations</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -132,6 +265,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.1</version>
+    </dependency>
+    <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-all</artifactId>
       <version>1.8.5</version>
@@ -140,14 +278,14 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop-hdfs.version}</version>
+      <version>${project.version}</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>
@@ -182,33 +320,38 @@
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-api</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-common</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-common</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
         <type>test-jar</type>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-server-common</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-server-nodemanager</artifactId>
-        <version>${yarn.version}</version>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.zookeeper</groupId>
@@ -256,26 +399,6 @@
           <version>2.4.1</version>
         </plugin>
         <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-surefire-plugin</artifactId>
-          <!-- requires 2.5+ to make system properties work -->
-          <!-- requires 2.7+ to avoid SUREFIRE-640 -->
-          <version>2.7.2</version>
-          <configuration>
-            <failIfNoTests>false</failIfNoTests>
-            <redirectTestOutputToFile>${test.logs}</redirectTestOutputToFile>
-            <forkedProcessTimeoutInSeconds>${test.timeout}</forkedProcessTimeoutInSeconds>
-            <environmentVariables>
-              <JAVA_HOME>${java.home}</JAVA_HOME>
-            </environmentVariables>
-            <systemPropertyVariables>
-              <build.dir>${project.build.directory}</build.dir>
-              <build.output.dir>${project.build.outputDirectory}</build.output.dir>
-              <log4j.configuration>file:///${yarn.basedir}/../src/test/log4j.properties</log4j.configuration>
-            </systemPropertyVariables>
-          </configuration>
-        </plugin>
-        <plugin>
           <groupId>com.atlassian.maven.plugins</groupId>
           <artifactId>maven-clover2-plugin</artifactId>
           <version>3.0.2</version>
@@ -316,66 +439,6 @@
       </plugins>
     </pluginManagement>
     <plugins>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <tarLongFileMode>gnu</tarLongFileMode>
-          <descriptors>
-            <descriptor>assembly/all.xml</descriptor>
-          </descriptors>
-        </configuration>
-      </plugin>
-      <plugin>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>santize-pom</id>
-            <phase>package</phase>
-            <configuration>
-              <target>
-                <echo message="project.build.directory: ${project.build.directory}"/>
-                <copy file="pom.xml" tofile="${install.pom}">
-                  <filterchain>
-                    <!-- we'll have to wait for ant 1.8.3 for the following
-                    <expandproperties>
-                      <propertyset regex=".*version$">
-                      </propertyset>
-                    </expandproperties>
-                    until then an even uglier workaround: -->
-                    <tokenfilter>
-                      <replaceregex pattern="\$\{hadoop-common.version}"
-                          replace="${hadoop-common.version}" flags="g"/>
-                      <replaceregex pattern="\$\{hadoop-hdfs.version}"
-                          replace="${hadoop-hdfs.version}" flags="g"/>
-                      <replaceregex pattern="\$\{yarn.version}"
-                          replace="${yarn.version}" flags="g"/>
-                    </tokenfilter>
-                  </filterchain>
-                </copy>
-              </target>
-            </configuration>
-            <goals>
-              <goal>run</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-install-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>install-sanitized-pom</id>
-            <configuration>
-              <file>${install.file}</file>
-              <pomFile>${install.pom}</pomFile>
-            </configuration>
-            <phase>install</phase>
-            <goals>
-              <goal>install-file</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
        <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>findbugs-maven-plugin</artifactId>

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml Wed Nov  2 05:34:31 2011
@@ -87,6 +87,8 @@
                rev="${yarn.version}" conf="compile->default"/>
    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
                rev="${yarn.version}" conf="compile->default"/>
+   <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common" 
+               rev="${yarn.version}" conf="compile->default"/>
    <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
                rev="${yarn.version}" conf="compile->default"/>
    <dependency org="log4j" name="log4j" rev="${log4j.version}" 
@@ -148,6 +150,7 @@
    <exclude org="javax.jms"/> 
    <exclude org="javax.mail"/> 
    <exclude org="org.apache.hadoop" module="avro"/> 
+   <exclude org="org.apache.commons" module="commons-daemon"/> 
 
  </dependencies>
   

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml Wed Nov  2 05:34:31 2011
@@ -15,9 +15,15 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>0.24.0-SNAPSHOT</version>
+    <relativePath>../hadoop-project</relativePath>
+  </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-mapreduce</artifactId>
-  <version>${hadoop-mapreduce.version}</version>
+  <version>0.24.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>hadoop-mapreduce</name>
   <url>http://hadoop.apache.org/mapreduce/</url>
@@ -26,35 +32,15 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <test.logs>true</test.logs>
     <test.timeout>600000</test.timeout>
-    <hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
-    <hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
-    <hadoop-mapreduce.version>0.24.0-SNAPSHOT</hadoop-mapreduce.version>
-    <yarn.version>0.24.0-SNAPSHOT</yarn.version>
-    <install.pom>${project.build.directory}/saner-pom.xml</install.pom>
-    <install.file>${install.pom}</install.file>
     <fork.mode>once</fork.mode>
     <mr.basedir>${basedir}</mr.basedir>
   </properties>
-
-  <repositories>
-    <repository>
-      <id>repository.jboss.org</id>
-      <url>http://repository.jboss.org/nexus/content/groups/public/</url>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-    </repository>
-    <repository>
-      <id>apache.snapshots</id>
-      <url>http://repository.apache.org/snapshots</url>
-      <!-- until we move to hadoop-common/hdfs trunk and/or maven 3 -->
-      <!-- cf. MNG-4326 -->
-      <snapshots>
-        <enabled>true</enabled>
-      </snapshots>
-    </repository>
-  </repositories>
-
+  
+  <modules>
+    <module>hadoop-yarn</module>
+    <module>hadoop-mapreduce-client</module>
+  </modules>
+  
   <dependencies>
     <dependency>
       <groupId>com.google.protobuf</groupId>
@@ -95,7 +81,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
@@ -120,7 +106,7 @@
         </exclusion>
       </exclusions>
     </dependency>
-
+ 
     <dependency>
      <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
@@ -134,7 +120,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-annotations</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
@@ -145,7 +131,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-common.version}</version>
+      <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
@@ -153,7 +139,7 @@
       <groupId>org.apache.hadoop</groupId>
       <!-- needed for security and runtime -->
       <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop-hdfs.version}</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>com.google.inject.extensions</groupId>
@@ -171,13 +157,18 @@
       <version>3.2.3.Final</version>
     </dependency>
     <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.1</version>
+    </dependency>
+    <dependency>
       <groupId>com.cenqua.clover</groupId>
       <artifactId>clover</artifactId>
       <version>3.0.2</version>
     </dependency>
-
+ 
   </dependencies>
-
+  
   <build>
     <pluginManagement>
       <plugins>
@@ -210,27 +201,6 @@
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-surefire-plugin</artifactId>
-          <!-- requires 2.5+ to make system properties work -->
-          <!-- requires 2.7+ to avoid SUREFIRE-640 -->
-          <version>2.7.2</version>
-          <configuration>
-            <failIfNoTests>false</failIfNoTests>
-            <redirectTestOutputToFile>${test.logs}</redirectTestOutputToFile>
-            <forkedProcessTimeoutInSeconds>${test.timeout}</forkedProcessTimeoutInSeconds>
-            <forkMode>${fork.mode}</forkMode>
-            <environmentVariables>
-              <JAVA_HOME>${java.home}</JAVA_HOME>
-            </environmentVariables>
-            <systemPropertyVariables>
-              <build.dir>${project.build.directory}</build.dir>
-              <build.output.dir>${project.build.outputDirectory}</build.output.dir>
-              <log4j.configuration>file:///${mr.basedir}/src/test/log4j.properties</log4j.configuration>  
-            </systemPropertyVariables>
-          </configuration>
-        </plugin>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-assembly-plugin</artifactId>
           <version>2.2.1</version>
         </plugin>
@@ -271,64 +241,35 @@
         <artifactId>maven-antrun-plugin</artifactId>
         <executions>
           <execution>
-            <id>santize-pom</id>
+            <id>tar</id>
             <phase>package</phase>
-            <configuration>
-              <target>
-                <copy file="pom.xml" tofile="${install.pom}">
-                  <filterchain>
-                    <!-- we'll have to wait for ant 1.8.3 for the following
-                    <expandproperties>
-                      <propertyset regex=".*version$">
-                      </propertyset>
-                    </expandproperties>
-                    until then an even uglier workaround: -->
-                    <tokenfilter>
-                      <replaceregex pattern="\$\{hadoop-common.version}"
-                          replace="${hadoop-common.version}" flags="g"/>
-                      <replaceregex pattern="\$\{hadoop-hdfs.version}"
-                          replace="${hadoop-hdfs.version}" flags="g"/>
-                      <replaceregex pattern="\$\{hadoop-mapreduce.version}"
-                          replace="${hadoop-mapreduce.version}" flags="g"/>
-                      <replaceregex pattern="\$\{yarn.version}"
-                          replace="${yarn.version}" flags="g"/>
-                    </tokenfilter>
-                  </filterchain>
-                </copy>
-              </target>
-            </configuration>
             <goals>
               <goal>run</goal>
             </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-install-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>install-sanitized-pom</id>
             <configuration>
-              <file>${install.file}</file>
-              <pomFile>${install.pom}</pomFile>
+              <!-- this is identical from hadoop-project-dist, eventually they must be unified -->
+              <target if="tar">
+                <!-- Using Unix script to preserve symlinks -->
+                <echo file="${project.build.directory}/dist-maketar.sh">
+
+                  which cygpath 2> /dev/null
+                  if [ $? = 1 ]; then
+                    BUILD_DIR="${project.build.directory}"
+                  else
+                    BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                  fi
+                  cd $BUILD_DIR
+                  tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+                </echo>
+                <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                  <arg line="./dist-maketar.sh"/>
+                </exec>
+              </target>
             </configuration>
-            <phase>install</phase>
-            <goals>
-              <goal>install-file</goal>
-            </goals>
           </execution>
         </executions>
       </plugin>
       <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <tarLongFileMode>gnu</tarLongFileMode>
-          <descriptors>
-            <descriptor>assembly/all.xml</descriptor>
-          </descriptors>
-        </configuration>
-      </plugin>
-      <plugin>
         <groupId>com.atlassian.maven.plugins</groupId>
         <artifactId>maven-clover2-plugin</artifactId>
         <executions>
@@ -380,14 +321,49 @@
             </executions>
           </plugin>
         </plugins>
+      </build>      
+    </profile>
+    <profile>
+      <id>dist</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-assemblies</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
+            <configuration>
+              <tarLongFileMode>gnu</tarLongFileMode>
+              <appendAssemblyId>false</appendAssemblyId>
+              <attach>false</attach>
+              <finalName>${project.artifactId}-${project.version}</finalName>
+              <descriptorRefs>
+                <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+              </descriptorRefs>
+            </configuration>
+            <executions>
+              <execution>
+                <id>dist</id>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
       </build>
     </profile>
   </profiles>
 
-  <modules>
-    <module>hadoop-yarn</module>
-    <module>hadoop-mapreduce-client</module>
-  </modules>
 
   <reporting>
     <plugins>

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112
 /hadoop/core/trunk/src/c++:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
 /hadoop/core/trunk/src/contrib:784664-785643

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1196451
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112
 /hadoop/core/trunk/src/contrib/block_forensics:784664-785643

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml Wed Nov  2 05:34:31 2011
@@ -60,13 +60,8 @@
     <subant target="test">
       <property name="continueOnFailure" value="true"/>
       <fileset dir="." includes="streaming/build.xml"/> 
-      <fileset dir="." includes="fairscheduler/build.xml"/> 
-      <fileset dir="." includes="capacity-scheduler/build.xml"/>  
-      <fileset dir="." includes="dynamic-scheduler/build.xml"/>
       <fileset dir="." includes="gridmix/build.xml"/>
       <fileset dir="." includes="vertica/build.xml"/>
-      <!-- mumak tests disabled due to timeouts. See MAPREDUCE-2348
-      <fileset dir="." includes="mumak/build.xml"/> -->
       <fileset dir="." includes="raid/build.xml"/>
     </subant>
     <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
@@ -88,20 +83,12 @@
                value="${hadoop.conf.dir.deployed}"/>
           <fileset dir="." includes="hdfsproxy/build.xml"/>
           <fileset dir="." includes="streaming/build.xml"/>
-          <fileset dir="." includes="fairscheduler/build.xml"/>
-         <fileset dir="." includes="capacity-scheduler/build.xml"/>
          <fileset dir="." includes="gridmix/build.xml"/>
       </subant>
       <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
       <fail if="testsfailed">Tests failed!</fail>
   </target>
 
-  <target name="docs">
-    <subant target="docs">
-      <fileset dir="." includes="capacity-scheduler/build.xml"/> 
-    </subant>
-  </target>
-
   <!-- ====================================================== -->
   <!-- Clean all the contribs.                              -->
   <!-- ====================================================== -->

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
 /hadoop/core/trunk/src/contrib/build.xml:776175-786373

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
 /hadoop/core/trunk/src/contrib/data_join:776175-786373

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1196451
 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112
 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Wed Nov  2 05:34:31 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.Mappe
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -72,7 +73,7 @@ class LoadJob extends GridmixJob {
           job.setNumReduceTasks(jobdesc.getNumberReduces());
           job.setMapOutputKeyClass(GridmixKey.class);
           job.setMapOutputValueClass(GridmixRecord.class);
-          job.setSortComparatorClass(GridmixKey.Comparator.class);
+          job.setSortComparatorClass(LoadSortComparator.class);
           job.setGroupingComparatorClass(SpecGroupingComparator.class);
           job.setInputFormatClass(LoadInputFormat.class);
           job.setOutputFormatClass(RawBytesOutputFormat.class);
@@ -94,17 +95,84 @@ class LoadJob extends GridmixJob {
   }
   
   /**
+   * This is a load matching key comparator which will make sure that the
+   * resource usage load is matched even when the framework is in control.
+   */
+  public static class LoadSortComparator extends GridmixKey.Comparator {
+    private ResourceUsageMatcherRunner matcher = null;
+    private boolean isConfigured = false;
+    
+    public LoadSortComparator() {
+      super();
+    }
+    
+    @Override
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+      configure();
+      int ret = super.compare(b1, s1, l1, b2, s2, l2);
+      if (matcher != null) {
+        try {
+          matcher.match(); // match the resource usage now
+        } catch (Exception e) {}
+      }
+      return ret;
+    }
+    
+    //TODO Note that the sorter will be instantiated 2 times as follows
+    //       1. During the sort/spill in the map phase
+    //       2. During the merge in the sort phase
+    // We need the handle to the matcher thread only in (2).
+    // This logic can be relaxed to run only in (2).
+    private void configure() {
+      if (!isConfigured) {
+        ThreadGroup group = Thread.currentThread().getThreadGroup();
+        Thread[] threads = new Thread[group.activeCount() * 2];
+        group.enumerate(threads, true);
+        for (Thread t : threads) {
+          if (t != null && (t instanceof ResourceUsageMatcherRunner)) {
+            this.matcher = (ResourceUsageMatcherRunner) t;
+            isConfigured = true;
+            break;
+          }
+        }
+      }
+    }
+  }
+  
+  /**
    * This is a progress based resource usage matcher.
    */
   @SuppressWarnings("unchecked")
-  static class ResourceUsageMatcherRunner extends Thread {
+  static class ResourceUsageMatcherRunner extends Thread 
+  implements Progressive {
     private final ResourceUsageMatcher matcher;
-    private final Progressive progress;
+    private final BoostingProgress progress;
     private final long sleepTime;
     private static final String SLEEP_CONFIG = 
       "gridmix.emulators.resource-usage.sleep-duration";
     private static final long DEFAULT_SLEEP_TIME = 100; // 100ms
     
+    /**
+     * This is a progress bar that can be boosted for weaker use-cases.
+     */
+    private static class BoostingProgress implements Progressive {
+      private float boostValue = 0f;
+      TaskInputOutputContext context;
+      
+      BoostingProgress(TaskInputOutputContext context) {
+        this.context = context;
+      }
+      
+      void setBoostValue(float boostValue) {
+        this.boostValue = boostValue;
+      }
+      
+      @Override
+      public float getProgress() {
+        return Math.min(1f, context.getProgress() + boostValue);
+      }
+    }
+    
     ResourceUsageMatcherRunner(final TaskInputOutputContext context, 
                                ResourceUsageMetrics metrics) {
       Configuration conf = context.getConfiguration();
@@ -118,19 +186,14 @@ class LoadJob extends GridmixJob {
       
       // set the other parameters
       this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
-      progress = new Progressive() {
-        @Override
-        public float getProgress() {
-          return context.getProgress();
-        }
-      };
+      progress = new BoostingProgress(context);
       
       // instantiate a resource-usage-matcher
       matcher = new ResourceUsageMatcher();
       matcher.configure(conf, plugin, metrics, progress);
     }
     
-    protected void match() throws Exception {
+    protected void match() throws IOException, InterruptedException {
       // match the resource usage
       matcher.matchResourceUsage();
     }
@@ -157,21 +220,34 @@ class LoadJob extends GridmixJob {
                  + " thread! Exiting.", e);
       }
     }
+    
+    @Override
+    public float getProgress() {
+      return matcher.getProgress();
+    }
+    
+    // boost the progress bar as fasten up the emulation cycles.
+    void boost(float value) {
+      progress.setBoostValue(value);
+    }
   }
   
   // Makes sure that the TaskTracker doesn't kill the map/reduce tasks while
   // they are emulating
   private static class StatusReporter extends Thread {
-    private TaskAttemptContext context;
-    StatusReporter(TaskAttemptContext context) {
+    private final TaskAttemptContext context;
+    private final Progressive progress;
+    
+    StatusReporter(TaskAttemptContext context, Progressive progress) {
       this.context = context;
+      this.progress = progress;
     }
     
     @Override
     public void run() {
       LOG.info("Status reporter thread started.");
       try {
-        while (context.getProgress() < 1) {
+        while (!isInterrupted() && progress.getProgress() < 1) {
           // report progress
           context.progress();
 
@@ -275,9 +351,11 @@ class LoadJob extends GridmixJob {
       
       matcher = new ResourceUsageMatcherRunner(ctxt, 
                       split.getMapResourceUsageMetrics());
+      matcher.setDaemon(true);
       
       // start the status reporter thread
-      reporter = new StatusReporter(ctxt);
+      reporter = new StatusReporter(ctxt, matcher);
+      reporter.setDaemon(true);
       reporter.start();
     }
 
@@ -324,6 +402,17 @@ class LoadJob extends GridmixJob {
         }
       }
       
+      // check if the thread will get a chance to run or not
+      //  check if there will be a sort&spill->merge phase or not
+      //  check if the final sort&spill->merge phase is gonna happen or not
+      if (context.getNumReduceTasks() > 0 
+          && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
+        LOG.info("Boosting the map phase progress.");
+        // add the sort phase progress to the map phase and emulate
+        matcher.boost(0.33f);
+        matcher.match();
+      }
+      
       // start the matcher thread since the map phase ends here
       matcher.start();
     }
@@ -392,7 +481,7 @@ class LoadJob extends GridmixJob {
       matcher = new ResourceUsageMatcherRunner(context, metrics);
       
       // start the status reporter thread
-      reporter = new StatusReporter(context);
+      reporter = new StatusReporter(context, matcher);
       reporter.start();
     }
     @Override
@@ -528,9 +617,13 @@ class LoadJob extends GridmixJob {
         specRecords[j] = info.getOutputRecords();
         metrics[j] = info.getResourceUsageMetrics();
         if (LOG.isDebugEnabled()) {
-          LOG.debug(String.format("SPEC(%d) %d -> %d %d %d", id(), i,
+          LOG.debug(String.format("SPEC(%d) %d -> %d %d %d %d %d %d %d", id(), i,
                     i + j * maps, info.getOutputRecords(), 
-                    info.getOutputBytes()));
+                    info.getOutputBytes(), 
+                    info.getResourceUsageMetrics().getCumulativeCpuUsage(),
+                    info.getResourceUsageMetrics().getPhysicalMemoryUsage(),
+                    info.getResourceUsageMetrics().getVirtualMemoryUsage(),
+                    info.getResourceUsageMetrics().getHeapUsage()));
         }
       }
       final TaskInfo info = jobdesc.getTaskInfo(TaskType.MAP, i);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java Wed Nov  2 05:34:31 2011
@@ -67,7 +67,7 @@ implements ResourceUsageEmulatorPlugin {
   private float emulationInterval; // emulation interval
   private long targetCpuUsage = 0;
   private float lastSeenProgress = 0;
-  private long lastSeenCpuUsageCpuUsage = 0;
+  private long lastSeenCpuUsage = 0;
   
   // Configuration parameters
   public static final String CPU_EMULATION_PROGRESS_INTERVAL = 
@@ -229,6 +229,15 @@ implements ResourceUsageEmulatorPlugin {
     return progress * progress * progress * progress;
   }
   
+  private synchronized long getCurrentCPUUsage() {
+    return monitor.getProcResourceValues().getCumulativeCpuTime();
+  }
+  
+  @Override
+  public float getProgress() {
+    return Math.min(1f, ((float)getCurrentCPUUsage())/targetCpuUsage);
+  }
+  
   @Override
   //TODO Multi-threading for speedup?
   public void emulate() throws IOException, InterruptedException {
@@ -249,10 +258,9 @@ implements ResourceUsageEmulatorPlugin {
         //   Note that (Cc-Cl)/(Pc-Pl) is termed as 'rate' in the following 
         //   section
         
-        long currentCpuUsage = 
-          monitor.getProcResourceValues().getCumulativeCpuTime();
+        long currentCpuUsage = getCurrentCPUUsage();
         // estimate the cpu usage rate
-        float rate = (currentCpuUsage - lastSeenCpuUsageCpuUsage)
+        float rate = (currentCpuUsage - lastSeenCpuUsage)
                      / (currentProgress - lastSeenProgress);
         long projectedUsage = 
           currentCpuUsage + (long)((1 - currentProgress) * rate);
@@ -264,8 +272,7 @@ implements ResourceUsageEmulatorPlugin {
             (long)(targetCpuUsage 
                    * getWeightForProgressInterval(currentProgress));
           
-          while (monitor.getProcResourceValues().getCumulativeCpuTime() 
-                 < currentWeighedTarget) {
+          while (getCurrentCPUUsage() < currentWeighedTarget) {
             emulatorCore.compute();
             // sleep for 100ms
             try {
@@ -281,8 +288,7 @@ implements ResourceUsageEmulatorPlugin {
         // set the last seen progress
         lastSeenProgress = progress.getProgress();
         // set the last seen usage
-        lastSeenCpuUsageCpuUsage = 
-          monitor.getProcResourceValues().getCumulativeCpuTime();
+        lastSeenCpuUsage = getCurrentCPUUsage();
       }
     }
   }
@@ -310,6 +316,6 @@ implements ResourceUsageEmulatorPlugin {
     
     // initialize the states
     lastSeenProgress = 0;
-    lastSeenCpuUsageCpuUsage = 0;
+    lastSeenCpuUsage = 0;
   }
 }
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java Wed Nov  2 05:34:31 2011
@@ -42,7 +42,7 @@ import org.apache.hadoop.conf.Configurat
  * For configuring GridMix to load and and use a resource usage emulator, 
  * see {@link ResourceUsageMatcher}. 
  */
-public interface ResourceUsageEmulatorPlugin {
+public interface ResourceUsageEmulatorPlugin extends Progressive {
   /**
    * Initialize the plugin. This might involve
    *   - initializing the variables

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java Wed Nov  2 05:34:31 2011
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.mapred.gridmix.emulators.resourceusage;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -35,7 +36,7 @@ import org.apache.hadoop.util.Reflection
  * <p>Note that the order in which the emulators are invoked is same as the 
  * order in which they are configured.
  */
-public class ResourceUsageMatcher {
+public class ResourceUsageMatcher implements Progressive {
   /**
    * Configuration key to set resource usage emulators.
    */
@@ -80,10 +81,31 @@ public class ResourceUsageMatcher {
     }
   }
   
-  public void matchResourceUsage() throws Exception {
+  public void matchResourceUsage() throws IOException, InterruptedException {
     for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
       // match the resource usage
       emulator.emulate();
     }
   }
+  
+  /**
+   * Returns the average progress.
+   */
+  @Override
+  public float getProgress() {
+    if (emulationPlugins.size() > 0) {
+      // return the average progress
+      float progress = 0f;
+      for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
+        // consider weighted progress of each emulator
+        progress += emulator.getProgress();
+      }
+
+      return progress / emulationPlugins.size();
+    }
+    
+    // if no emulators are configured then return 1
+    return 1f;
+    
+  }
 }
\ No newline at end of file

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Wed Nov  2 05:34:31 2011
@@ -187,6 +187,11 @@ implements ResourceUsageEmulatorPlugin {
   }
   
   @Override
+  public float getProgress() {
+    return Math.min(1f, ((float)getTotalHeapUsageInMB())/targetHeapUsageInMB);
+  }
+  
+  @Override
   public void emulate() throws IOException, InterruptedException {
     if (enabled) {
       float currentProgress = progress.getProgress();

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java Wed Nov  2 05:34:31 2011
@@ -135,6 +135,14 @@ public class TestResourceUsageEmulators 
              ? fs.getFileStatus(testPath).getModificationTime() 
              : 0;
     }
+    
+    @Override
+    public float getProgress() {
+      try {
+        return fs.exists(touchPath) ? 1.0f : 0f;
+      } catch (IOException ioe) {}
+      return 0f;
+    }
   }
   
   /**

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
 /hadoop/core/trunk/src/contrib/index:776175-786373

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java Wed Nov  2 05:34:31 2011
@@ -782,7 +782,7 @@ public abstract class BlockFixer extends
         DatanodeInfo[] nodes = new DatanodeInfo[]{datanode};
         new Sender(out).writeBlock(block.getBlock(), block.getBlockToken(), "",
             nodes, null, BlockConstructionStage.PIPELINE_SETUP_CREATE,
-            1, 0L, blockSize, 0L);
+            1, 0L, blockSize, 0L, DataChecksum.newDataChecksum(metadataIn));
         blockSender.sendBlock(out, baseStream);
         
         LOG.info("Sent block " + block.getBlock() + " to " + datanode.name);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java Wed Nov  2 05:34:31 2011
@@ -73,8 +73,6 @@ import org.apache.hadoop.mapreduce.lib.o
  * raid.blockfix.filespertask       - number of corrupt files to fix in a single
  *                                    map reduce task (i.e., at one mapper node)
  *
- * raid.blockfix.fairscheduler.pool - the pool to use for block fixer jobs
- *
  * raid.blockfix.maxpendingfiles    - maximum number of files to fix 
  *                                    simultaneously
  */
@@ -91,12 +89,6 @@ public class DistBlockFixer extends Bloc
     "raid.blockfix.filespertask";
   private static final String BLOCKFIX_MAX_PENDING_FILES =
     "raid.blockfix.maxpendingfiles";
-  private static final String BLOCKFIX_POOL = 
-    "raid.blockfix.fairscheduler.pool";
-  // mapred.fairscheduler.pool is only used in the local configuration
-  // passed to a block fixing job
-  private static final String MAPRED_POOL = 
-    "mapred.fairscheduler.pool";
 
   // default number of files to fix in a task
   private static final long DEFAULT_BLOCKFIX_FILES_PER_TASK = 10L;
@@ -115,9 +107,6 @@ public class DistBlockFixer extends Bloc
   // number of files being fixed right now
   private long pendingFiles;
 
-  // pool name to use (may be null, in which case no special pool is used)
-  private String poolName;
-
   private long lastCheckTime;
 
   private final SimpleDateFormat dateFormat = 
@@ -137,7 +126,6 @@ public class DistBlockFixer extends Bloc
     filesPerTask = DistBlockFixer.filesPerTask(getConf());
     maxPendingFiles = DistBlockFixer.maxPendingFiles(getConf());
     pendingFiles = 0L;
-    poolName = conf.get(BLOCKFIX_POOL);
 
     // start off due for the first iteration
     lastCheckTime = System.currentTimeMillis() - blockFixInterval;
@@ -365,9 +353,6 @@ public class DistBlockFixer extends Bloc
     List<Path> filesInJob = createInputFile(jobName, inDir, corruptFiles);
 
     Configuration jobConf = new Configuration(getConf());
-    if (poolName != null) {
-      jobConf.set(MAPRED_POOL, poolName);
-    }
     Job job = new Job(jobConf, jobName);
     job.setJarByClass(getClass());
     job.setMapperClass(DistBlockFixerMapper.class);

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java Wed Nov  2 05:34:31 2011
@@ -693,14 +693,4 @@ public class TestRaidNode extends TestCa
     }
     LOG.info("Test testSuspendTraversal completed.");
   }
-
-  public void testSchedulerOption() throws IOException {
-    Configuration conf = new Configuration();
-    conf.set("raid.scheduleroption",
-      "mapred.fairscheduler.pool:dummy,foo:bar");
-    org.apache.hadoop.mapreduce.Job job = DistRaid.createJob(conf);
-    Configuration jobConf = job.getConfiguration();
-    assertEquals("dummy", jobConf.get("mapred.fairscheduler.pool"));
-    assertEquals("bar", jobConf.get("foo"));
-  }
 }

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/streaming:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/streaming:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
 /hadoop/core/trunk/src/contrib/streaming:776175-786373

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java Wed Nov  2 05:34:31 2011
@@ -718,7 +718,7 @@ public class StreamJob implements Tool {
     }
 
     // general MapRed job properties
-    jobConf_ = new JobConf(config_);
+    jobConf_ = new JobConf(config_, StreamJob.class);
     
     // All streaming jobs get the task timeout value
     // from the configuration settings.

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
 /hadoop/core/trunk/src/contrib/vaidya:776175-786373

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml Wed Nov  2 05:34:31 2011
@@ -1232,49 +1232,28 @@
            </li>
          
            <li>
-           <code>mapreduce.{map|reduce}.ulimit</code>: The slaves where
-           tasks are run could be configured with a ulimit value that
-           applies a limit to every process that is launched on the slave.
-           If the task, or any child that the task launches (like in
-           streaming), requires more than the configured limit, this option
-           must be used. The value is given in kilobytes. For example, to
-           increase the ulimit to 1G, the option should be set to 1048576.
-           Note that this value is a per process limit. Since it applies 
-           to the JVM as well, the heap space given to the JVM through 
-           the <code>mapreduce.{map|reduce}.java.opts</code> should be less
-           than the value configured for the ulimit. Otherwise the JVM
-           will not start.
-           </li>
-           
-           <li>
-           <code>mapreduce.{map|reduce}.memory.mb</code>: In some 
-           environments, administrators might have configured a total limit
-           on the virtual memory used by the entire process tree for a task, 
-           including all processes launched recursively by the task or 
-           its children, like in streaming. More details about this can be
-           found in the section on 
-           <a href="ext:cluster-setup/ConfiguringMemoryParameters">
-           Monitoring Task Memory Usage</a> in the Cluster SetUp guide.
-           If a task requires more virtual memory for its entire tree, 
-           this option 
-           must be used. The value is given in MB. For example, to set 
+           <code>mapreduce.{map|reduce}.memory.mb</code>:
+           This parameter configures how many megabytes of physical memory
+           the job requires for its map and reduce tasks. It must be configured
+           to be slightly larger than the configured Java heap size above,
+           to account for the fact that the JVM uses physical memory beyond
+           just the heap. An overhead of 20% to 30% is usually sufficient.
+           <br/>
+           Note that, the smaller the amount of memory specified for tasks
+           in this configuration, the larger the number of tasks that
+           can be run in parallel.<br/>
+
+           The value is given in MB. For example, to set 
            the limit to 1G, the option should be set to 1024. Note that this 
            value does not automatically influence the per process ulimit or 
            heap space. Hence, you may need to set those parameters as well 
            (as described above) in order to give your tasks the right amount 
            of memory.
+           <br/>
+           If the amount of physical memory used by your task exceeds the
+           configured value, the NodeManager will automatically kill the task.
            </li>
           
-           <li>
-           <code>mapreduce.{map|reduce}.memory.physical.mb</code>: 
-           This parameter is similar to 
-           <code>mapreduce.{map|reduce}.memory.mb</code>, except it specifies
-           how much physical memory is required by a task for its entire
-           tree of processes. The parameter is applicable if administrators
-           have configured a total limit on the physical memory used by
-           all MapReduce tasks.
-           </li>
- 
          </ul>
          
          <p>
@@ -1297,7 +1276,7 @@
          <p>
          Note: The memory related configuration options described above 
          are used only for configuring the launched child tasks from the 
-         tasktracker. Configuring the memory options for daemons is documented 
+         NodeManager. Configuring the memory options for daemons is documented 
          under
          <a href="ext:cluster-setup/ConfiguringEnvironmentHadoopDaemons">
          Configuring the Environment of the Hadoop Daemons</a> (Cluster Setup).

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml Wed Nov  2 05:34:31 2011
@@ -43,15 +43,9 @@ See http://forrest.apache.org/docs/linki
 		<vaidya    					label="Vaidya" 		href="vaidya.html"/>
 		<archives  				label="Hadoop Archives"     href="hadoop_archives.html"/>
 		<gridmix  				label="Gridmix"     href="gridmix.html"/>
-		<Rumen  				label="Rumen"     href="rumen.html"/>
+                <Rumen                                  label="Rumen"     href="rumen.html"/>
    </docs>
    
-    <docs label="Schedulers">
-        <cap_scheduler 		label="Capacity Scheduler"     href="capacity_scheduler.html"/>
-		<fair_scheduler 			label="Fair Scheduler"            href="fair_scheduler.html"/>
-		<dyn_scheduler 			label="Dynamic Priority Scheduler"            href="dynamic_scheduler.html"/>
-    </docs>
-   
    <docs label="Miscellaneous"> 
 		<api       	label="API Docs"           href="ext:api/index" />
 		<jdiff     	label="API Changes"      href="ext:jdiff/changes" />
@@ -75,8 +69,6 @@ See http://forrest.apache.org/docs/linki
     <hdfs-default href="http://hadoop.apache.org/hdfs/docs/current/hdfs-default.html" />
     <mapred-default href="http://hadoop.apache.org/mapreduce/docs/current/mapred-default.html" />
     <mapred-queues href="http://hadoop.apache.org/mapreduce/docs/current/mapred-queues.xml" />
-    <mapred-queues-capacity-scheduler href="http://hadoop.apache.org/mapreduce/docs/current/mapred-queues-capacity-scheduler.xml" />
-    <capacity-scheduler-conf href="http://hadoop.apache.org/mapreduce/docs/current/capacity-scheduler-conf.html" />
 
     <single-node-setup href="http://hadoop.apache.org/common/docs/current/single_node_setup.html">
       <PreReqs href="#PreReqs" />

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112
 /hadoop/core/trunk/src/examples:776175-784663

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java Wed Nov  2 05:34:31 2011
@@ -29,7 +29,6 @@ import org.apache.hadoop.mapreduce.JobCo
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 
@@ -100,28 +99,9 @@ public class TeraOutputFormat extends Fi
       throws IOException {
     if (committer == null) {
       Path output = getOutputPath(context);
-      committer = new TeraOutputCommitter(output, context);
+      committer = new FileOutputCommitter(output, context);
     }
     return committer;
   }
 
-  public static class TeraOutputCommitter extends FileOutputCommitter {
-
-    public TeraOutputCommitter(Path outputPath, TaskAttemptContext context)
-        throws IOException {
-      super(outputPath, context);
-    }
-
-    @Override
-    public void commitJob(JobContext jobContext) {
-    }
-
-    @Override
-    public void setupJob(JobContext jobContext) {
-    }
-
-    @Override
-    public void setupTask(TaskAttemptContext taskContext) {
-    }
-  }
 }

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/java:713112
 /hadoop/core/trunk/src/mapred:776175-785643

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider Wed Nov  2 05:34:31 2011
@@ -12,4 +12,3 @@
 #   limitations under the License.
 #
 org.apache.hadoop.mapred.JobTrackerClientProtocolProvider
-org.apache.hadoop.mapred.LocalClientProtocolProvider

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Nov  2 05:34:31 2011
@@ -2676,7 +2676,7 @@ public class JobInProgress {
 
     TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
         status.getTaskID(), taskType, status.getStartTime(), 
-        status.getTaskTracker(),  ttStatus.getHttpPort());
+        status.getTaskTracker(),  ttStatus.getHttpPort(), -1);
     
     jobHistory.logEvent(tse, status.getTaskID().getJobID());
     TaskAttemptID statusAttemptID = status.getTaskID();
@@ -2685,7 +2685,7 @@ public class JobInProgress {
       MapAttemptFinishedEvent mfe = new MapAttemptFinishedEvent(
           statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(),
           status.getMapFinishTime(),
-          status.getFinishTime(),  trackerHostname,
+          status.getFinishTime(),  trackerHostname, "",
           status.getStateString(), 
           new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
           tip.getSplits(statusAttemptID).burst()
@@ -2698,7 +2698,7 @@ public class JobInProgress {
           statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(), 
           status.getShuffleFinishTime(),
           status.getSortFinishTime(), status.getFinishTime(),
-          trackerHostname, status.getStateString(),
+          trackerHostname, "", status.getStateString(),
           new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
           tip.getSplits(statusAttemptID).burst()
           );
@@ -3197,7 +3197,7 @@ public class JobInProgress {
       StringUtils.arrayToString(taskDiagnosticInfo.toArray(new String[0]));
     TaskType taskType = getTaskType(tip);
     TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
-        taskid, taskType, startTime, taskTrackerName, taskTrackerPort);
+        taskid, taskType, startTime, taskTrackerName, taskTrackerPort, -1);
     
     jobHistory.logEvent(tse, taskid.getJobID());
 

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java Wed Nov  2 05:34:31 2011
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.serve
 import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
 import org.apache.hadoop.mapreduce.util.ConfigUtil;
 import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
+import org.apache.hadoop.mapreduce.v2.LogParams;
 import org.apache.hadoop.net.DNSToSwitchMapping;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NetworkTopology;
@@ -4829,6 +4830,13 @@ public class JobTracker implements MRCon
     return secretManager.renewToken(token, user);
   }
 
+  @Override
+  public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
+      org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
+      throws IOException, InterruptedException {
+    throw new UnsupportedOperationException("Not supported by JobTracker");
+  }
+  
   JobACLsManager getJobACLsManager() {
     return aclsManager.getJobACLsManager();
   }

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java Wed Nov  2 05:34:31 2011
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
 import org.apache.hadoop.security.RefreshUserMappingsProtocol;
 import org.apache.hadoop.security.authorize.PolicyProvider;
@@ -40,14 +41,17 @@ public class MapReducePolicyProvider ext
                   ClientProtocol.class),
       new Service("security.task.umbilical.protocol.acl", 
                   TaskUmbilicalProtocol.class),
-      new Service("security.refresh.policy.protocol.acl", 
-                  RefreshAuthorizationPolicyProtocol.class),
-      new Service("security.refresh.user.mappings.protocol.acl", 
-                  RefreshUserMappingsProtocol.class),
+      new Service(
+          CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY, 
+          RefreshAuthorizationPolicyProtocol.class),
+      new Service(
+          CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_USER_MAPPINGS, 
+          RefreshUserMappingsProtocol.class),
       new Service("security.admin.operations.protocol.acl", 
                   AdminOperationsProtocol.class),
-      new Service("security.get.user.mappings.protocol.acl",
-                  GetUserMappingsProtocol.class)
+      new Service(
+          CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_GET_USER_MAPPINGS,
+          GetUserMappingsProtocol.class)
   };
   
   @Override

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/
            ('svn:externals' removed)

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties Wed Nov  2 05:34:31 2011
@@ -16,4 +16,4 @@ log4j.rootLogger=info,stdout
 log4j.threshhold=ALL
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
 /hadoop/core/trunk/src/test/mapred:776175-785643

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java Wed Nov  2 05:34:31 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.security.author
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.xml.sax.SAXException;
 
@@ -113,6 +114,7 @@ public class TestMRCLI extends TestHDFSC
   }
 
   @Test
+  @Ignore
   @Override
   public void testAll () {
     super.testAll();

Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java Wed Nov  2 05:34:31 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.conf;
 import junit.framework.Assert;
 
 import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileUtil;
@@ -59,7 +60,18 @@ public class TestNoDefaultsJobConf exten
     JobConf conf = new JobConf(false);
 
     //seeding JT and NN info into non-defaults (empty jobconf)
-    conf.set(JTConfig.JT_IPC_ADDRESS, createJobConf().get(JTConfig.JT_IPC_ADDRESS));
+    String jobTrackerAddress = createJobConf().get(JTConfig.JT_IPC_ADDRESS);
+    if (jobTrackerAddress == null) {
+      jobTrackerAddress = "local";
+    }
+    conf.set(JTConfig.JT_IPC_ADDRESS, jobTrackerAddress);
+    if (jobTrackerAddress == "local") {
+      conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+    }
+    else {
+      conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);      
+    }
+    
     conf.set("fs.default.name", createJobConf().get("fs.default.name"));
 
     conf.setJobName("mr");

Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov  2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1196451
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934



Mime
View raw message