hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r952462 - in /hadoop/mapreduce/branches/branch-0.21: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/lib/
Date Mon, 07 Jun 2010 22:14:21 GMT
Author: tomwhite
Date: Mon Jun  7 22:14:20 2010
New Revision: 952462

URL: http://svn.apache.org/viewvc?rev=952462&view=rev
Log:
Merge -r 952459:952460 from trunk to branch-0.21. Fixes: MAPREDUCE-1725

Modified:
    hadoop/mapreduce/branches/branch-0.21/CHANGES.txt   (contents, props changed)
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java

Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Jun  7 22:14:20 2010
@@ -1584,3 +1584,6 @@ Release 0.21.0 - Unreleased
     MAPREDUCE-913. TaskRunner crashes with NPE resulting in held up slots,
     UNINITIALIZED tasks and hung TaskTracker. (Amareshwari Sriramadasu and
     Sreekanth Ramakrishnan via vinodkv)
+
+    MAPREDUCE-1725.  Fix MapReduce API incompatibilities between 0.20 and 0.21.
+    (tomwhite)

Propchange: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jun  7 22:14:20 2010
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
 /hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
-/hadoop/mapreduce/trunk/CHANGES.txt:940364,942764,946526
+/hadoop/mapreduce/trunk/CHANGES.txt:940364,942764,946526,952460

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
Mon Jun  7 22:14:20 2010
@@ -26,9 +26,9 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
 
 /** A base class for {@link OutputFormat}. */
 @InterfaceAudience.Public

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
Mon Jun  7 22:14:20 2010
@@ -63,6 +63,22 @@ public class TaskAttemptID extends org.a
    * Constructs a TaskId object from given parts.
    * @param jtIdentifier jobTracker identifier
    * @param jobId job number 
+   * @param isMap whether the tip is a map 
+   * @param taskId taskId number
+   * @param id the task attempt number
+   * @deprecated Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.
+   */
+  @Deprecated
+  public TaskAttemptID(String jtIdentifier, int jobId, boolean isMap, 
+      int taskId, int id) {
+    this(jtIdentifier, jobId, isMap ? TaskType.MAP : TaskType.REDUCE, taskId,
+	id);
+  }
+  
+  /**
+   * Constructs a TaskId object from given parts.
+   * @param jtIdentifier jobTracker identifier
+   * @param jobId job number 
    * @param type the TaskType 
    * @param taskId taskId number
    * @param id the task attempt number
@@ -122,6 +138,31 @@ public class TaskAttemptID extends org.a
    * of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first

    * map task</i>, we would use :
    * <pre> 
+   * TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+   * </pre>
+   * which will return :
+   * <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+   * @param jtIdentifier jobTracker identifier, or null
+   * @param jobId job number, or null
+   * @param isMap whether the tip is a map, or null 
+   * @param taskId taskId number, or null
+   * @param attemptId the task attempt number, or null
+   * @return a regex pattern matching TaskAttemptIDs
+   */
+  @Deprecated
+  public static String getTaskAttemptIDsPattern(String jtIdentifier,
+      Integer jobId, Boolean isMap, Integer taskId, Integer attemptId) {
+    return getTaskAttemptIDsPattern(jtIdentifier, jobId,
+	isMap ? TaskType.MAP : TaskType.REDUCE, taskId, attemptId);
+  }
+  
+  /** 
+   * Returns a regex pattern which matches task attempt IDs. Arguments can 
+   * be given null, in which case that part of the regex will be generic.  
+   * For example to obtain a regex matching <i>all task attempt IDs</i> 
+   * of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first

+   * map task</i>, we would use :
+   * <pre> 
    * TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
    * </pre>
    * which will return :

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java Mon
Jun  7 22:14:20 2010
@@ -55,6 +55,32 @@ public class TaskID extends org.apache.h
   /**
    * Constructs a TaskID object from given {@link JobID}.  
    * @param jobId JobID that this tip belongs to 
+   * @param isMap whether the tip is a map 
+   * @param id the tip number
+   * @deprecated Use {@link #TaskID(String, int, TaskType, int)}
+   */
+  @Deprecated
+  public TaskID(org.apache.hadoop.mapreduce.JobID jobId, boolean isMap,int id) {
+    this(jobId, isMap ? TaskType.MAP : TaskType.REDUCE, id);
+  }
+   
+  /**
+   * Constructs a TaskInProgressId object from given parts.
+   * @param jtIdentifier jobTracker identifier
+   * @param jobId job number 
+   * @param isMap whether the tip is a map 
+   * @param id the tip number
+   * @deprecated Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+   * int)}
+   */
+  @Deprecated
+  public TaskID(String jtIdentifier, int jobId, boolean isMap, int id) {
+    this(jtIdentifier, jobId, isMap ? TaskType.MAP : TaskType.REDUCE, id);
+  }
+    
+  /**
+   * Constructs a TaskID object from given {@link JobID}.  
+   * @param jobId JobID that this tip belongs to 
    * @param type the {@link TaskType} 
    * @param id the tip number
    */
@@ -114,6 +140,31 @@ public class TaskID extends org.apache.h
    * <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
    * @param jtIdentifier jobTracker identifier, or null
    * @param jobId job number, or null
+   * @param isMap whether the tip is a map, or null 
+   * @param taskId taskId number, or null
+   * @return a regex pattern matching TaskIDs
+   * @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+   * Integer)}
+   */
+  @Deprecated
+  public static String getTaskIDsPattern(String jtIdentifier, Integer jobId
+      , Boolean isMap, Integer taskId) {
+    return getTaskIDsPattern(jtIdentifier, jobId,
+	isMap ? TaskType.MAP : TaskType.REDUCE, taskId);
+  }
+  
+  /** 
+   * Returns a regex pattern which matches task IDs. Arguments can 
+   * be given null, in which case that part of the regex will be generic.  
+   * For example to obtain a regex matching <i>the first map task</i> 
+   * of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+   * <pre> 
+   * TaskID.getTaskIDsPattern(null, null, true, 1);
+   * </pre>
+   * which will return :
+   * <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+   * @param jtIdentifier jobTracker identifier, or null
+   * @param jobId job number, or null
    * @param type the {@link TaskType}, or null 
    * @param taskId taskId number, or null
    * @return a regex pattern matching TaskIDs

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
Mon Jun  7 22:14:20 2010
@@ -19,9 +19,11 @@
 package org.apache.hadoop.mapred.lib;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
@@ -68,6 +70,27 @@ public abstract class CombineFileInputFo
     throws IOException {
     return super.getSplits(new Job(job)).toArray(new InputSplit[0]);
   }
+  
+  /**
+   * Create a new pool and add the filters to it.
+   * A split cannot have files from different pools.
+   * @deprecated Use {@link #createPool(List)}.
+   */
+  @Deprecated
+  protected void createPool(JobConf conf, List<PathFilter> filters) {
+    createPool(filters);
+  }
+
+  /**
+   * Create a new pool and add the filters to it. 
+   * A pathname can satisfy any one of the specified filters.
+   * A split cannot have files from different pools.
+   * @deprecated Use {@link #createPool(PathFilter...)}.
+   */
+  @Deprecated
+  protected void createPool(JobConf conf, PathFilter... filters) {
+    createPool(filters);
+  }
 
   /**
    * This is not implemented yet. 

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java
Mon Jun  7 22:14:20 2010
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.mapred.lib;
 
+import java.io.IOException;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
@@ -39,11 +41,24 @@ public class CombineFileSplit extends 
 
   public CombineFileSplit() {
   }
+  
+  public CombineFileSplit(JobConf job, Path[] files, long[] start, 
+          long[] lengths, String[] locations) {
+    super(files, start, lengths, locations);
+    this.job = job;
+  }
 
   public CombineFileSplit(JobConf job, Path[] files, long[] lengths) {
     super(files, lengths);
     this.job = job;
   }
+  
+  /**
+   * Copy constructor
+   */
+  public CombineFileSplit(CombineFileSplit old) throws IOException {
+    super(old);
+  }
 
   public JobConf getJob() {
     return job;



Mime
View raw message