hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1580983 - in /hadoop/common/branches/branch-1: CHANGES.txt src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
Date Mon, 24 Mar 2014 18:42:15 GMT
Author: cnauroth
Date: Mon Mar 24 18:42:15 2014
New Revision: 1580983

URL: http://svn.apache.org/r1580983
Log:
MAPREDUCE-5808. Port output replication factor configurable for terasort to Hadoop 1.x. Contributed
by Chuan Liu.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1580983&r1=1580982&r2=1580983&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Mar 24 18:42:15 2014
@@ -202,6 +202,9 @@ Release 1.3.0 - unreleased
     HDFS-5516. WebHDFS does not require user name when anonymous http requests
     are disallowed. (Miodrag Radulovic via cnauroth)
 
+    MAPREDUCE-5808. Port output replication factor configurable for terasort to
+    Hadoop 1.x. (Chuan Liu via cnauroth)
+
 Release 1.2.2 - unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1580983&r1=1580982&r2=1580983&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
(original)
+++ hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
Mon Mar 24 18:42:15 2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.util.ToolRunner
  */
 public class TeraSort extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(TeraSort.class);
+  private static final String OUTPUT_REPLICATION = "mapreduce.terasort.output.replication";
 
   /**
    * A partitioner that splits text keys into roughly equal partitions
@@ -222,6 +223,10 @@ public class TeraSort extends Configured
     }
     
   }
+
+  public static int getOutputReplication(JobConf job) {
+    return job.getInt(OUTPUT_REPLICATION, 1);
+  }
   
   public int run(String[] args) throws Exception {
     LOG.info("starting");
@@ -243,7 +248,7 @@ public class TeraSort extends Configured
     TeraInputFormat.writePartitionFile(job, partitionFile);
     DistributedCache.addCacheFile(partitionUri, job);
     DistributedCache.createSymlink(job);
-    job.setInt("dfs.replication", 1);
+    job.setInt("dfs.replication", getOutputReplication(job));
     TeraOutputFormat.setFinalSync(job, true);
     JobClient.runJob(job);
     LOG.info("done");



Mime
View raw message