hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1153931 - in /hadoop/common/branches/HDFS-1623/mapreduce: ./ conf/ src/c++/ src/contrib/ src/contrib/block_forensics/ src/contrib/capacity-scheduler/ src/contrib/data_join/ src/contrib/dynamic-scheduler/ src/contrib/eclipse-plugin/ src/con...
Date Thu, 04 Aug 2011 17:09:59 GMT
Author: todd
Date: Thu Aug  4 17:09:14 2011
New Revision: 1153931

URL: http://svn.apache.org/viewvc?rev=1153931&view=rev
Log:
Merge trunk into HA branch after mavenization of hadoop-common

(no conflicts, straight SVN merge)

Added:
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java
      - copied unchanged from r1153927, hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java
Modified:
    hadoop/common/branches/HDFS-1623/mapreduce/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/.gitignore   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/conf/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/conf/capacity-scheduler.xml.template   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/c++/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/block_forensics/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build-contrib.xml   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build.xml   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/examples/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/fs/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/hdfs/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/ipc/   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java   (props changed)
    hadoop/common/branches/HDFS-1623/mapreduce/src/webapps/job/   (props changed)

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1 +1,2 @@
+/hadoop/common/trunk/mapreduce:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred:713112

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/.gitignore:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
 /hadoop/core/trunk/.gitignore:784664-785643

Modified: hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt Thu Aug  4 17:09:14 2011
@@ -40,6 +40,9 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
+    acmurthy) 
+
     MAPREDUCE-2365. Add counters to track bytes (read,written) via 
     File(Input,Output)Format. (Siddharth Seth via acmurthy)
  
@@ -206,12 +209,18 @@ Trunk (unreleased changes)
     MAPREDUCE-2602. Allow setting of end-of-record delimiter for
     TextInputFormat for the old API. (Ahmed Radwan via todd)
 
+    MAPREDUCE-2705. Permits parallel multiple task launches. 
+    (Thomas Graves via ddas)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and
     JobInProgress.getCounters() aquire locks in a shorter time period.
     (Joydeep Sen Sarma via schen)
 
+    MAPREDUCE-2740. MultipleOutputs in new API creates needless
+    TaskAttemptContexts. (todd)
+
   BUG FIXES
 
     MAPREDUCE-2603. Disable High-Ram emulation in system tests. 
@@ -362,6 +371,16 @@ Trunk (unreleased changes)
     MAPREDUCE-2463. Job history files are not moved to done folder when job
     history location is hdfs.  (Devaraj K via szetszwo)
 
+    MAPREDUCE-2243. Close streams propely in a finally-block to avoid leakage
+    in CompletedJobStatusStore, TaskLog, EventWriter and TotalOrderPartitioner.
+    (Devaraj K via szetszwo)
+
+    MAPREDUCE-2741. Make ant build system work with hadoop-common JAR
+    generated by Maven. (Alejandro Abdelnur via tomwhite)
+
+    MAPREDUCE-2760. mapreduce.jobtracker.split.metainfo.maxsize typoed
+    in mapred-default.xml. (todd via eli)
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/CHANGES.txt:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
 /hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/conf:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/conf:713112
 /hadoop/core/trunk/conf:784664-785643

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/conf/capacity-scheduler.xml.template:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
 /hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643

Modified: hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml Thu Aug  4 17:09:14 2011
@@ -57,6 +57,7 @@
     <artifact conf="master"/>
   </publications>
  <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="compile->default"/>
    <dependency org="org.apache.hadoop" name="hadoop-common" 
                rev="${hadoop-common.version}" conf="compile->default"/> 
    <dependency org="org.apache.hadoop" name="hadoop-common-test" 
@@ -105,6 +106,8 @@
       <exclude module="jetty"/>
       <exclude module="slf4j-simple"/>
     </dependency>
+   <dependency org="junit" name="junit" rev="${junit.version}"
+               conf="test->default"/>
    <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}" 
                conf="test->default"/>
    <dependency org="org.vafer" name="jdeb" rev="${jdeb.version}" conf="package->master"/>
@@ -116,6 +119,12 @@
    <dependency org="org.aspectj" name="aspectjtools" rev="${aspectj.version}"
                conf="compile->default"/>
 
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
  </dependencies>
   
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/c++:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112
 /hadoop/core/trunk/src/c++:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
 /hadoop/core/trunk/src/contrib:784664-785643

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1 +1,2 @@
+/hadoop/common/trunk/mapreduce/src/contrib/block_forensics:1152502-1153927
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/build-contrib.xml:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/build.xml:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
 /hadoop/core/trunk/src/contrib/build.xml:776175-786373

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/capacity-scheduler:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/capacity-scheduler:713112
 /hadoop/core/trunk/src/contrib/capacity-scheduler:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml Thu Aug  4 17:09:14 2011
@@ -41,6 +41,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" 
                 rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common-test" 
@@ -75,5 +76,12 @@
       <exclude module="jetty"/>
       <exclude module="slf4j-simple"/>
     </dependency>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/data_join:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
 /hadoop/core/trunk/src/contrib/data_join:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml Thu Aug  4 17:09:14 2011
@@ -41,11 +41,20 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
      <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}" conf="common->default"/>
      <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
      <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop-hdfs.version}" conf="common->default"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
+    <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/dynamic-scheduler:1152502-1153927
 /hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
 /hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml Thu Aug  4 17:09:14 2011
@@ -37,6 +37,7 @@
   </publications>
 
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master"/>
     <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}" conf="common->master"/>
@@ -50,5 +51,12 @@
     <dependency org="org.mortbay.jetty" name="servlet-api-2.5" rev="${servlet-api-2.5.version}" conf="common->master"/>
     <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
     <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}" conf="common->master"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/eclipse-plugin:1152502-1153927
 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml Thu Aug  4 17:09:14 2011
@@ -41,9 +41,17 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/fairscheduler:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/fairscheduler:713112
 /hadoop/core/trunk/src/contrib/fairscheduler:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml Thu Aug  4 17:09:14 2011
@@ -40,6 +40,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" 
                 rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common-test" 
@@ -100,5 +101,12 @@
       name="jsp-2.1"
       rev="${jetty.version}"
       conf="common->master"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml Thu Aug  4 17:09:14 2011
@@ -36,6 +36,10 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+    <dependency org="org.apache.hadoop" 
+      name="hadoop-annotations"
+      rev="${hadoop-common.version}"
+      conf="common->default"/>
     <dependency org="org.apache.hadoop"
       name="hadoop-common"
       rev="${hadoop-common.version}"
@@ -122,5 +126,12 @@
       name="paranamer"
       rev="${paranamer.version}"
       conf="common->default"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/index:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
 /hadoop/core/trunk/src/contrib/index:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml Thu Aug  4 17:09:14 2011
@@ -41,6 +41,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
      <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}" conf="common->default"/>
      <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
@@ -48,5 +49,13 @@
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
     <dependency org="org.apache.lucene" name="lucene-core" rev="${lucene-core.version}" conf="common->default"/>
+    <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml Thu Aug  4 17:09:14 2011
@@ -39,6 +39,10 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+    <dependency org="org.apache.hadoop" 
+      name="hadoop-annotations"
+      rev="${hadoop-common.version}"
+      conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" 
                 rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common-test" 
@@ -122,5 +126,12 @@
       <exclude module="jetty"/>
       <exclude module="slf4j-simple"/>
     </dependency>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml Thu Aug  4 17:09:14 2011
@@ -36,6 +36,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop"
       name="hadoop-common"
       rev="${hadoop-common.version}"
@@ -122,5 +123,12 @@
       name="paranamer"
       rev="${paranamer.version}"
       conf="common->default"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/> 
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/streaming:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
 /hadoop/core/trunk/src/contrib/streaming:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml Thu Aug  4 17:09:14 2011
@@ -40,6 +40,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" 
                 rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common-test" 
@@ -76,5 +77,12 @@
     </dependency>
     <dependency org="org.slf4j" name="slf4j-api" 
                 rev="${slf4j-api.version}" conf="common->master"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/>
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/vaidya:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
 /hadoop/core/trunk/src/contrib/vaidya:776175-786373

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml Thu Aug  4 17:09:14 2011
@@ -41,8 +41,16 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/>
+
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml Thu Aug  4 17:09:14 2011
@@ -40,6 +40,7 @@
     <artifact conf="master"/>
   </publications>
   <dependencies>
+   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
     <dependency org="org.apache.hadoop"
       name="hadoop-common"
       rev="${hadoop-common.version}"
@@ -64,5 +65,12 @@
       name="log4j"
       rev="${log4j.version}"
       conf="common->master"/>
+
+   <!-- Exclusions for transitive dependencies pulled in by log4j -->
+   <exclude org="com.sun.jdmk"/>
+   <exclude org="com.sun.jmx"/>
+   <exclude org="javax.jms"/> 
+   <exclude org="javax.mail"/>
+
   </dependencies>
 </ivy-module>

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/examples:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112
 /hadoop/core/trunk/src/examples:776175-784663

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/java:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/java:713112
 /hadoop/core/trunk/src/mapred:776175-785643

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml Thu Aug  4 17:09:14 2011
@@ -256,7 +256,7 @@
 
 
 <property>
-  <name>mapreduce.job.split.metainfo.maxsize</name>
+  <name>mapreduce.jobtracker.split.metainfo.maxsize</name>
   <value>10000000</value>
   <description>The maximum permissible size of the split metainfo file. 
   The JobTracker won't attempt to read split metainfo files bigger than
@@ -1042,6 +1042,14 @@
 <!-- End of TaskTracker DistributedCache configuration -->
 
 <property>
+  <name>mapreduce.task.combine.progress.records</name>
+  <value>10000</value>
+  <description> The number of records to process during combine output collection 
+   before sending a progress notification to the TaskTracker.
+  </description>
+</property>
+
+<property>
   <name>mapreduce.task.merge.progress.records</name>
   <value>10000</value>
   <description> The number of records to process during merge before

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java Thu Aug  4 17:09:14 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.security.UserGr
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 
 /**
@@ -172,8 +173,9 @@ class CompletedJobStatusStore implements
     if (active && retainTime > 0) {
       JobID jobId = job.getStatus().getJobID();
       Path jobStatusFile = getInfoFilePath(jobId);
+      FSDataOutputStream dataOut = null;
       try {
-        FSDataOutputStream dataOut = fs.create(jobStatusFile);
+        dataOut = fs.create(jobStatusFile);
 
         job.getStatus().write(dataOut);
 
@@ -189,6 +191,7 @@ class CompletedJobStatusStore implements
         }
 
         dataOut.close();
+        dataOut = null; // set dataOut to null explicitly so that close in finally will not be executed again.
       } catch (IOException ex) {
         LOG.warn("Could not store [" + jobId + "] job info : " +
                  ex.getMessage(), ex);
@@ -198,6 +201,8 @@ class CompletedJobStatusStore implements
         catch (IOException ex1) {
           //ignore
         }
+      } finally {
+        IOUtils.cleanup(LOG, dataOut);
       }
     }
   }

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java Thu Aug  4 17:09:14 2011
@@ -946,7 +946,7 @@ class MapTask extends Task {
       if (combinerRunner != null) {
         final Counters.Counter combineOutputCounter =
           reporter.getCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
-        combineCollector= new CombineOutputCollector<K,V>(combineOutputCounter);
+        combineCollector= new CombineOutputCollector<K,V>(combineOutputCounter, reporter, conf);
       } else {
         combineCollector = null;
       }

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java Thu Aug  4 17:09:14 2011
@@ -352,7 +352,7 @@ public class ReduceTask extends Task {
       Class combinerClass = conf.getCombinerClass();
       CombineOutputCollector combineCollector = 
         (null != combinerClass) ? 
-            new CombineOutputCollector(reduceCombineOutputCounter) : null;
+ 	     new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null;
 
       Shuffle shuffle = 
         new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical, 

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java Thu Aug  4 17:09:14 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapreduce.Outpu
 import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
 import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
 import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
@@ -79,6 +80,7 @@ abstract public class Task implements Wr
     LogFactory.getLog(Task.class);
 
   public static String MERGED_OUTPUT_PREFIX = ".merged";
+  public static final long DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS = 10000;
 
   /**
    * Counters to measure the usage of the different file systems.
@@ -1176,16 +1178,26 @@ abstract public class Task implements Wr
   implements OutputCollector<K, V> {
     private Writer<K, V> writer;
     private Counters.Counter outCounter;
-    public CombineOutputCollector(Counters.Counter outCounter) {
+    private Progressable progressable;
+    private long progressBar;
+
+    public CombineOutputCollector(Counters.Counter outCounter, Progressable progressable, Configuration conf) {
       this.outCounter = outCounter;
+      this.progressable=progressable;
+      progressBar = conf.getLong(MRJobConfig.COMBINE_RECORDS_BEFORE_PROGRESS, DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS);
     }
+    
     public synchronized void setWriter(Writer<K, V> writer) {
       this.writer = writer;
     }
+
     public synchronized void collect(K key, V value)
         throws IOException {
       outCounter.increment(1);
       writer.append(key, value);
+      if ((outCounter.getValue() % progressBar) == 0) {
+        progressable.progress();
+      }
     }
   }
 

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java Thu Aug  4 17:09:14 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.LocalFileSys
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SecureIOUtils;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.util.ProcessTree;
@@ -111,34 +112,42 @@ public class TaskLog {
     //stderr:<start-offset in the stderr file> <length>
     //syslog:<start-offset in the syslog file> <length>
     LogFileDetail l = new LogFileDetail();
-    String str = fis.readLine();
-    if (str == null) { //the file doesn't have anything
-      throw new IOException ("Index file for the log of " + taskid+" doesn't exist.");
-    }
-    l.location = str.substring(str.indexOf(LogFileDetail.LOCATION)+
-        LogFileDetail.LOCATION.length());
-    //special cases are the debugout and profile.out files. They are guaranteed
-    //to be associated with each task attempt since jvm reuse is disabled
-    //when profiling/debugging is enabled
-    if (filter.equals(LogName.DEBUGOUT) || filter.equals(LogName.PROFILE)) {
-      l.length = new File(l.location, filter.toString()).length();
-      l.start = 0;
-      fis.close();
-      return l;
-    }
-    str = fis.readLine();
-    while (str != null) {
-      //look for the exact line containing the logname
-      if (str.contains(filter.toString())) {
-        str = str.substring(filter.toString().length()+1);
-        String[] startAndLen = str.split(" ");
-        l.start = Long.parseLong(startAndLen[0]);
-        l.length = Long.parseLong(startAndLen[1]);
-        break;
+    String str = null;
+    try {
+      str = fis.readLine();
+      if (str == null) { // the file doesn't have anything
+        throw new IOException("Index file for the log of " + taskid
+            + " doesn't exist.");
+      }
+      l.location = str.substring(str.indexOf(LogFileDetail.LOCATION)
+          + LogFileDetail.LOCATION.length());
+      // special cases are the debugout and profile.out files. They are
+      // guaranteed
+      // to be associated with each task attempt since jvm reuse is disabled
+      // when profiling/debugging is enabled
+      if (filter.equals(LogName.DEBUGOUT) || filter.equals(LogName.PROFILE)) {
+        l.length = new File(l.location, filter.toString()).length();
+        l.start = 0;
+        fis.close();
+        return l;
       }
       str = fis.readLine();
+      while (str != null) {
+        // look for the exact line containing the logname
+        if (str.contains(filter.toString())) {
+          str = str.substring(filter.toString().length() + 1);
+          String[] startAndLen = str.split(" ");
+          l.start = Long.parseLong(startAndLen[0]);
+          l.length = Long.parseLong(startAndLen[1]);
+          break;
+        }
+        str = fis.readLine();
+      }
+      fis.close();
+      fis = null;
+    } finally {
+      IOUtils.cleanup(LOG, fis);
     }
-    fis.close();
     return l;
   }
   
@@ -189,22 +198,27 @@ public class TaskLog {
     //LOG_DIR: <the dir where the task logs are really stored>
     //STDOUT: <start-offset in the stdout file> <length>
     //STDERR: <start-offset in the stderr file> <length>
-    //SYSLOG: <start-offset in the syslog file> <length>    
-    dos.writeBytes(LogFileDetail.LOCATION + logLocation + "\n"
-        + LogName.STDOUT.toString() + ":");
-    dos.writeBytes(Long.toString(prevOutLength) + " ");
-    dos.writeBytes(Long.toString(new File(logLocation, LogName.STDOUT
-        .toString()).length() - prevOutLength)
-        + "\n" + LogName.STDERR + ":");
-    dos.writeBytes(Long.toString(prevErrLength) + " ");
-    dos.writeBytes(Long.toString(new File(logLocation, LogName.STDERR
-        .toString()).length() - prevErrLength)
-        + "\n" + LogName.SYSLOG.toString() + ":");
-    dos.writeBytes(Long.toString(prevLogLength) + " ");
-    dos.writeBytes(Long.toString(new File(logLocation, LogName.SYSLOG
-        .toString()).length() - prevLogLength)
-        + "\n");
-    dos.close();
+    //SYSLOG: <start-offset in the syslog file> <length>   
+    try{
+      dos.writeBytes(LogFileDetail.LOCATION + logLocation + "\n"
+          + LogName.STDOUT.toString() + ":");
+      dos.writeBytes(Long.toString(prevOutLength) + " ");
+      dos.writeBytes(Long.toString(new File(logLocation, LogName.STDOUT
+          .toString()).length() - prevOutLength)
+          + "\n" + LogName.STDERR + ":");
+      dos.writeBytes(Long.toString(prevErrLength) + " ");
+      dos.writeBytes(Long.toString(new File(logLocation, LogName.STDERR
+          .toString()).length() - prevErrLength)
+          + "\n" + LogName.SYSLOG.toString() + ":");
+      dos.writeBytes(Long.toString(prevLogLength) + " ");
+      dos.writeBytes(Long.toString(new File(logLocation, LogName.SYSLOG
+          .toString()).length() - prevLogLength)
+          + "\n");
+      dos.close();
+      dos = null;
+    } finally {
+      IOUtils.cleanup(LOG, dos);
+    }
 
     File indexFile = getIndexFile(currentTaskid, isCleanup);
     Path indexFilePath = new Path(indexFile.getAbsolutePath());

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu Aug  4 17:09:14 2011
@@ -1253,7 +1253,7 @@ public class TaskTracker 
     }
   }
 
-  private void launchTaskForJob(TaskInProgress tip, JobConf jobConf,
+  protected void launchTaskForJob(TaskInProgress tip, JobConf jobConf,
       UserGroupInformation ugi) throws IOException {
     synchronized (tip) {
       tip.setJobConf(jobConf);
@@ -2351,30 +2351,35 @@ public class TaskTracker 
    * All exceptions are handled locally, so that we don't mess up the
    * task tracker.
    */
-  void startNewTask(TaskInProgress tip) {
-    try {
-      RunningJob rjob = localizeJob(tip);
-      // Localization is done. Neither rjob.jobConf nor rjob.ugi can be null
-      launchTaskForJob(tip, new JobConf(rjob.jobConf), rjob.ugi); 
-    } catch (Throwable e) {
-      String msg = ("Error initializing " + tip.getTask().getTaskID() + 
-                    ":\n" + StringUtils.stringifyException(e));
-      LOG.warn(msg);
-      tip.reportDiagnosticInfo(msg);
-      try {
-        tip.kill(true);
-        tip.cleanup(true);
-      } catch (IOException ie2) {
-        LOG.info("Error cleaning up " + tip.getTask().getTaskID() + ":\n" +
-                 StringUtils.stringifyException(ie2));          
+  void startNewTask(final TaskInProgress tip) {
+    Thread launchThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          RunningJob rjob = localizeJob(tip);
+          // Localization is done. Neither rjob.jobConf nor rjob.ugi can be null
+          launchTaskForJob(tip, new JobConf(rjob.getJobConf()), rjob.ugi); 
+        } catch (Throwable e) {
+          String msg = ("Error initializing " + tip.getTask().getTaskID() + 
+                        ":\n" + StringUtils.stringifyException(e));
+          LOG.warn(msg);
+          tip.reportDiagnosticInfo(msg);
+          try {
+            tip.kill(true);
+            tip.cleanup(true);
+          } catch (IOException ie2) {
+            LOG.info("Error cleaning up " + tip.getTask().getTaskID() + ":\n" +
+                     StringUtils.stringifyException(ie2));          
+          }
+          if (e instanceof Error) {
+            LOG.error("TaskLauncher error " +
+                StringUtils.stringifyException(e));
+          }
+        }
       }
+    });
+    launchThread.start();
         
-      // Careful! 
-      // This might not be an 'Exception' - don't handle 'Error' here!
-      if (e instanceof Error) {
-        throw ((Error) e);
-      }
-    }
   }
   
   void addToMemoryManager(TaskAttemptID attemptId, boolean isMap, 

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java Thu Aug  4 17:09:14 2011
@@ -260,6 +260,8 @@ public interface MRJobConfig {
 
   public static final String REDUCE_MEMTOMEM_ENABLED = "mapreduce.reduce.merge.memtomem.enabled";
 
+  public static final String COMBINE_RECORDS_BEFORE_PROGRESS = "mapreduce.task.combine.progress.records";
+
   public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers";
 
   public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal";

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Thu Aug  4 17:09:14 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.CounterGroup;
 import org.apache.hadoop.mapreduce.Counters;
@@ -33,6 +34,8 @@ import org.apache.avro.io.DatumWriter;
 import org.apache.avro.specific.SpecificDatumWriter;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 /**
  * Event Writer is an utility class used to write events to the underlying
@@ -47,6 +50,7 @@ class EventWriter {
   private DatumWriter<Event> writer =
     new SpecificDatumWriter<Event>(Event.class);
   private Encoder encoder;
+  private static final Log LOG = LogFactory.getLog(EventWriter.class);
   
   EventWriter(FSDataOutputStream out) throws IOException {
     this.out = out;
@@ -72,8 +76,13 @@ class EventWriter {
   }
 
   void close() throws IOException {
-    encoder.flush();
-    out.close();
+    try {
+      encoder.flush();
+      out.close();
+      out = null;
+    } finally {
+      IOUtils.cleanup(LOG, out);
+    }
   }
 
   private static final Schema GROUPS =

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java Thu Aug  4 17:09:14 2011
@@ -132,6 +132,10 @@ public class MultipleOutputs<KEYOUT, VAL
    * Cache for the taskContexts
    */
   private Map<String, TaskAttemptContext> taskContexts = new HashMap<String, TaskAttemptContext>();
+  /**
+   * Cached TaskAttemptContext which uses the job's configured settings
+   */
+  private TaskAttemptContext jobOutputFormatContext;
 
   /**
    * Checks if a named output name is valid token.
@@ -382,11 +386,13 @@ public class MultipleOutputs<KEYOUT, VAL
   public void write(KEYOUT key, VALUEOUT value, String baseOutputPath) 
       throws IOException, InterruptedException {
     checkBaseOutputPath(baseOutputPath);
-    TaskAttemptContext taskContext = 
-      new TaskAttemptContextImpl(context.getConfiguration(), 
-                                 context.getTaskAttemptID(),
-                                 new WrappedStatusReporter(context));
-    getRecordWriter(taskContext, baseOutputPath).write(key, value);
+    if (jobOutputFormatContext == null) {
+      jobOutputFormatContext = 
+        new TaskAttemptContextImpl(context.getConfiguration(), 
+                                   context.getTaskAttemptID(),
+                                   new WrappedStatusReporter(context));
+    }
+    getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value);
   }
 
   // by being synchronized MultipleOutputTask can be use with a

Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java Thu Aug  4 17:09:14 2011
@@ -23,6 +23,8 @@ import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.Arrays;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
@@ -30,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.RawComparator;
@@ -56,6 +59,7 @@ public class TotalOrderPartitioner<K ext
   public static final String NATURAL_ORDER = 
     "mapreduce.totalorderpartitioner.naturalorder";
   Configuration conf;
+  private static final Log LOG = LogFactory.getLog(TotalOrderPartitioner.class);
 
   public TotalOrderPartitioner() { }
 
@@ -298,11 +302,16 @@ public class TotalOrderPartitioner<K ext
     ArrayList<K> parts = new ArrayList<K>();
     K key = ReflectionUtils.newInstance(keyClass, conf);
     NullWritable value = NullWritable.get();
-    while (reader.next(key, value)) {
-      parts.add(key);
-      key = ReflectionUtils.newInstance(keyClass, conf);
+    try {
+      while (reader.next(key, value)) {
+        parts.add(key);
+        key = ReflectionUtils.newInstance(keyClass, conf);
+      }
+      reader.close();
+      reader = null;
+    } finally {
+      IOUtils.cleanup(LOG, reader);
     }
-    reader.close();
     return parts.toArray((K[])Array.newInstance(keyClass, parts.size()));
   }
   

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
 /hadoop/core/trunk/src/test/mapred:776175-785643

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/fs:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/hdfs:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/ipc:1152502-1153927
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
 /hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs-with-mr/org/apache/hadoop/ipc:796829-820463

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
 /hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934

Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug  4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/webapps/job:1152502-1153927
 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
 /hadoop/core/trunk/src/webapps/job:776175-785643



Mime
View raw message