tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces. Removes Tez configuration for in-degree and out-degree. (sseth)
Date Thu, 23 May 2013 20:40:59 GMT
Updated Branches:
  refs/heads/TEZ-1 268f3ac97 -> 4c2732c99


TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces.
Removes Tez configuration for in-degree and out-degree. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c2732c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c2732c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c2732c9

Branch: refs/heads/TEZ-1
Commit: 4c2732c991a4cc76011acedbf9c17726c06bc145
Parents: 268f3ac
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu May 23 13:40:22 2013 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu May 23 13:40:22 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/common/TezJobConfig.java   |   16 +---------
 .../org/apache/hadoop/mapred/LocalJobRunner.java   |    1 -
 .../tez/mapreduce/hadoop/DeprecatedKeys.java       |   17 ++++------
 .../processor/reduce/ReduceProcessor.java          |    1 -
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |    4 ++
 .../tez/mapreduce/hadoop/TestDeprecatedKeys.java   |   23 +-------------
 6 files changed, 14 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 0d6f6be..c0b0ae6 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -66,7 +66,7 @@ public class TezJobConfig {
    */
   public static final String LOCAL_DIRS = "tez.engine.local.dirs";
   public static final String DEFAULT_LOCAL_DIRS = "/tmp";
-  
+
   /**
    * One local dir for the speicfic job.
    */
@@ -80,20 +80,6 @@ public class TezJobConfig {
   public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
   
   public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
-  
-  /**
-   * 
-   */
-  public static final String TEZ_ENGINE_TASK_INDEGREE = 
-      "tez.engine.task.in-degree";
-  public static final int DEFAULT_TEZ_ENGINE_TASK_INDEGREE = 1;
-
-  /**
-   * 
-   */
-  public static final String TEZ_ENGINE_TASK_OUTDEGREE = 
-      "tez.engine.task.out-degree";
-  public static final int DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE = 1;
 
   /**
    * 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5c181e2..5dade2b 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -445,7 +445,6 @@ public class LocalJobRunner implements ClientProtocol {
             JobConf localConf = new JobConf(job);
             localConf.setUser(user);
             localConf.set("mapreduce.jobtracker.address", "local");
-            localConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, mapIds.size());
             setupChildMapredLocalDirs(reduceId, user, localConf);
 
             // FIXME invalid task context

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 01d4fe0..499159f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -136,8 +136,6 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_ENGINE_IFILE_READAHEAD_BYTES);
     
     registerMRToEngineKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.RECORDS_BEFORE_PROGRESS);
-    
-    registerMRToEngineKeyTranslation(MRJobConfig.JOB_LOCAL_DIR, MRConfig.LOCAL_DIR);
 
     registerMRToEngineKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR);
     
@@ -149,6 +147,12 @@ public class DeprecatedKeys {
     
     registerMRToEngineKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS);
     
+    // Counter replacement will work in this manner, as long as TezCounters
+    // extends MRCounters and is used directly by the Mapper/Reducer.
+    // When these counters are eventually translated over to MRCounters, this
+    // may break.
+    // Framework counters, like FILESYSTEM will likely be incompatible since
+    // they enum key belongs to a different package.
     registerMRToEngineKeyTranslation(MRJobConfig.COUNTERS_MAX_KEY, TezJobConfig.COUNTERS_MAX_KEY);
     
     registerMRToEngineKeyTranslation(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY);
@@ -180,7 +184,7 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM);
     
     registerMRToEngineKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_ENGINE_INPUT_BUFFER_PERCENT);
-    
+
     registerMRToEngineKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.DAG_CREDENTIALS_BINARY);
     
     registerMRToEngineKeyTranslation("map.sort.class", TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS);
@@ -188,17 +192,10 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_GROUP_COMPARATOR_CLASS);
     
     registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
-    
-    registerMRToEngineKeyTranslation(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
 
-    registerMRToEngineKeyTranslation(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
   }
   
   private static void addDeprecatedKeys() {
-    // Adding deprecation for num_maps, num_reduces - in case some component in
-    // Hadoop MR land uses these. Tez components already use tez-in/out-degree.
-    _(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
-    _(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
   }
 
   private static void registerMRToEngineKeyTranslation(String mrKey,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index ce49726..9a6fb5b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -338,7 +338,6 @@ implements Processor {
       throws IOException, InterruptedException {
     super.localizeConfiguration(jobConf);
     jobConf.setBoolean(JobContext.TASK_ISMAP, false);
-    jobConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, numMapTasks); 
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 34aaa1b..a3d5684 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -170,6 +170,10 @@ public class MRRuntimeTask extends RuntimeTask {
 
     job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
     job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
+    if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) {
+      job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+          job.get(TezJobConfig.DAG_CREDENTIALS_BINARY));
+    }
 
     // setup the child's attempt directories
     // Do the task-type specific localization

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 16e783e..99aad74 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -18,28 +18,9 @@
 
 package org.apache.tez.mapreduce.hadoop;
 
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.tez.common.TezJobConfig;
-import org.junit.Test;
 
 public class TestDeprecatedKeys {
 
-  @Test
-  public void testNumTasksDeprecation() {
-    
-    Configuration conf = new Configuration(false);
-    conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, 6);
-    conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 3);
-    
-    
-    DeprecatedKeys.init();
-    JobConf jobConf = new JobConf(conf);
-    
-    Assert.assertEquals(6, jobConf.getNumMapTasks());
-    Assert.assertEquals(3, jobConf.getNumReduceTasks());
-  }
-  
+  // TODO Add tests
+
 }


Mime
View raw message