tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject svn commit: r1470656 - in /incubator/tez/branches/TEZ-1: tez-dag-api/src/main/java/org/apache/tez/dag/api/ tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/ tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/
Date Mon, 22 Apr 2013 18:31:41 GMT
Author: sseth
Date: Mon Apr 22 18:31:41 2013
New Revision: 1470656

URL: http://svn.apache.org/r1470656
Log:
TEZ-57. Remove MR references from AMNode. (sseth)

Modified:
    incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
    incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java

Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java?rev=1470656&r1=1470655&r2=1470656&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
Mon Apr 22 18:31:41 2013
@@ -88,6 +88,8 @@ public class DAGConfiguration extends Co
   public static final String JOB_NAME = DAG + "job.name";
   public static final String USER_NAME = DAG + "user.name";
   
+  // TODO Some of the DAG properties are job specific and not AM specific. Rename accordingly.
+  
   // TODO Speculator class should be configurable on a pere vertex level.
   public static final String DAG_AM_SPECULATOR_CLASS = DAG_AM + "speculator.class";
   
@@ -97,6 +99,15 @@ public class DAGConfiguration extends Co
   public static final String DAG_AM_STAGING_DIR = DAG_AM + "staging-dir";
   public static final String DAG_AM_STAGING_DIR_DEFAULT = "/tmp/hadoop-yarn/staging";
   
+  // TODO Are any of these node blacklisting properties required. (other than for MR compat)
+  public static final String DAG_MAX_TASK_FAILURES_PER_NODE = DAG
+      + "maxtaskfailures.per.node";
+  public static final String DAG_NODE_BLACKLISTING_ENABLED = DAG
+      + "node-blacklisting.enabled";
+  public static final String DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD = DAG
+      + "node-blacklisting.ignore-threshold-node-percent";
+  public static final int DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
+  
   @Private
   public void setEdgeProperties(List<Edge> edges) {
     String[] edgeIds = new String[edges.size()];

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java?rev=1470656&r1=1470655&r2=1470656&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
Mon Apr 22 18:31:41 2013
@@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.tez.dag.api.DAGConfiguration;
 import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -65,12 +65,12 @@ public class AMNodeMap extends AbstractS
   @Override
   public synchronized void init(Configuration conf) {
     this.maxTaskFailuresPerNode = conf.getInt(
-        MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 3);
+        DAGConfiguration.DAG_MAX_TASK_FAILURES_PER_NODE, 3);
     this.nodeBlacklistingEnabled = conf.getBoolean(
-        MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, true);
+        DAGConfiguration.DAG_NODE_BLACKLISTING_ENABLED, true);
     this.blacklistDisablePercent = conf.getInt(
-          MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
-          MRJobConfig.DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT);
+          DAGConfiguration.DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD,
+          DAGConfiguration.DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT);
 
     LOG.info("blacklistDisablePercent is " + blacklistDisablePercent +
         ", blacklistingEnabled: " + nodeBlacklistingEnabled + 

Modified: incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java?rev=1470656&r1=1470655&r2=1470656&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
Mon Apr 22 18:31:41 2013
@@ -40,17 +40,29 @@ public class DeprecatedKeys {
   static {
     addDeprecatedKeys();
     
-    mrParamToDAGParamMap.put(MRJobConfig.JOB_SUBMIT_DIR, DAGConfiguration.JOB_SUBMIT_DIR);
-    mrParamToDAGParamMap.put(MRJobConfig.APPLICATION_TOKENS_FILE, DAGConfiguration.APPLICATION_TOKENS_FILE);
-    
+    mrParamToDAGParamMap.put(MRJobConfig.JOB_SUBMIT_DIR,
+        DAGConfiguration.JOB_SUBMIT_DIR);
+    mrParamToDAGParamMap.put(MRJobConfig.APPLICATION_TOKENS_FILE,
+        DAGConfiguration.APPLICATION_TOKENS_FILE);
+
     mrParamToDAGParamMap.put(MRJobConfig.JOB_NAME, DAGConfiguration.JOB_NAME);
-    
-    mrParamToDAGParamMap.put(MRJobConfig.MR_AM_JOB_SPECULATOR, DAGConfiguration.DAG_AM_SPECULATOR_CLASS);
-    
+
+    mrParamToDAGParamMap.put(MRJobConfig.MR_AM_JOB_SPECULATOR,
+        DAGConfiguration.DAG_AM_SPECULATOR_CLASS);
+
     // TODO Default value handling.
-    mrParamToDAGParamMap.put(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, DAGConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT);
+    mrParamToDAGParamMap.put(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT,
+        DAGConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT);
     
     mrParamToDAGParamMap.put(MRJobConfig.USER_NAME, DAGConfiguration.USER_NAME);
+    
+    mrParamToDAGParamMap.put(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER,
+        DAGConfiguration.DAG_MAX_TASK_FAILURES_PER_NODE);
+    mrParamToDAGParamMap.put(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE,
+        DAGConfiguration.DAG_NODE_BLACKLISTING_ENABLED);
+    mrParamToDAGParamMap.put(
+        MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
+        DAGConfiguration.DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD);
   }
 
   // TODO TEZAM4 Sometime, make sure this gets loaded by default. Insteaf of the current
initialization in MRAppMaster, TezChild.



Mime
View raw message