tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject svn commit: r1471669 - in /incubator/tez/branches/TEZ-1: tez-dag-api/src/main/java/org/apache/tez/dag/api/ tez-dag/src/main/java/org/apache/hadoop/mapred/ tez-dag/src/main/java/org/apache/tez/dag/app/ tez-dag/src/main/java/org/apache/tez/dag/app/rm/con...
Date Wed, 24 Apr 2013 20:51:54 GMT
Author: hitesh
Date: Wed Apr 24 20:51:53 2013
New Revision: 1471669

URL: http://svn.apache.org/r1471669
Log:
TEZ-77. Fix build break due to changes in YARN related to ApplicationConstants.Environment.
(hitesh)

Modified:
    incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
    incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java

Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
Wed Apr 24 20:51:53 2013
@@ -91,9 +91,6 @@ public class TezConfiguration extends Co
     TEZ_HOME_ENV + "/lib/*"
   };
 
-  public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV";
-  
-  
   public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml";
 
 }

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
Wed Apr 24 20:51:53 2013
@@ -60,6 +60,7 @@ import org.apache.hadoop.util.StringUtil
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -105,9 +106,9 @@ public class YarnTezDagChild {
     final InetSocketAddress address =
         NetUtils.createSocketAddrForHost(host, port);
 
-    final ApplicationId appID = ConverterUtils.toApplicationId(args[2]);
-    
-    final ContainerId containerId = ConverterUtils.toContainerId(args[3]);
+    final ContainerId containerId = ConverterUtils.toContainerId(args[2]);
+    final ApplicationId appID =
+        containerId.getApplicationAttemptId().getApplicationId();
 
     // FIXME fix initialize metrics in child runner
     DefaultMetricsSystem.initialize("VertexTask");
@@ -172,7 +173,8 @@ public class YarnTezDagChild {
         taskAttemptId = taskContext.getTaskAttemptId();
 
         final Task t = createAndConfigureTezTask(taskContext, umbilical,
-            credentials, jt);
+            credentials, jt,
+            containerId.getApplicationAttemptId().getAttemptId());
         task = (MRTask) t.getProcessor();
         final JobConf job = task.getConf();
 
@@ -251,7 +253,7 @@ public class YarnTezDagChild {
    */
   private static void configureLocalDirs(MRTask task, JobConf job) throws IOException {
     String[] localSysDirs = StringUtils.getTrimmedStrings(
-        System.getenv(ApplicationConstants.LOCAL_DIR_ENV));
+        System.getenv(Environment.LOCAL_DIRS.name()));
     job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
     LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
         job.get(TezJobConfig.LOCAL_DIR));
@@ -289,15 +291,12 @@ public class YarnTezDagChild {
   }
 
   private static JobConf configureTask(MRTask task, Credentials credentials,
-      Token<JobTokenIdentifier> jt) throws IOException, InterruptedException {
+      Token<JobTokenIdentifier> jt, int appAttemptId)
+          throws IOException, InterruptedException {
     JobConf job = task.getConf();
     
-    String appAttemptIdEnv = System
-        .getenv(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV);
-    LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv);
     // Set it in conf, so as to be able to be used the the OutputCommitter.
-    job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer
-        .parseInt(appAttemptIdEnv));
+    job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
 
     // set tcp nodelay
     job.setBoolean("ipc.client.tcpnodelay", true);
@@ -406,8 +405,9 @@ public class YarnTezDagChild {
 
   private static Task createAndConfigureTezTask(
       TezEngineTaskContext taskContext,
-      TezTaskUmbilicalProtocol master, 
-      Credentials credentials, Token<JobTokenIdentifier> jt) 
+      TezTaskUmbilicalProtocol master,
+      Credentials credentials, Token<JobTokenIdentifier> jt,
+      int appAttemptId)
       throws IOException, InterruptedException {
     Configuration jConf = new JobConf(MRJobConfig.JOB_CONF_FILE);
     Configuration conf;
@@ -466,7 +466,7 @@ public class YarnTezDagChild {
     t.initialize(job, master);
     
     MRTask task = (MRTask)t.getProcessor();
-    configureTask(task, credentials, jt);
+    configureTask(task, credentials, jt, appAttemptId);
     
     return t;
   }

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
Wed Apr 24 20:51:53 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.SystemCloc
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -1239,20 +1240,14 @@ public class DAGAppMaster extends Compos
       // TODO: Deprecated keys?
       //DeprecatedKeys.init();
       String containerIdStr =
-          System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV);
-      String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV);
-      String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV);
+          System.getenv(Environment.CONTAINER_ID.name());
+      String nodeHostString = System.getenv(Environment.NM_HOST.name());
+      String nodePortString = System.getenv(Environment.NM_PORT.name());
       String nodeHttpPortString =
-          System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV);
+          System.getenv(Environment.NM_HTTP_PORT.name());
       String appSubmitTimeStr =
           System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
 
-      validateInputParam(containerIdStr,
-          ApplicationConstants.AM_CONTAINER_ID_ENV);
-      validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV);
-      validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV);
-      validateInputParam(nodeHttpPortString,
-          ApplicationConstants.NM_HTTP_PORT_ENV);
       validateInputParam(appSubmitTimeStr,
           ApplicationConstants.APP_SUBMIT_TIME_ENV);
 

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
Wed Apr 24 20:51:53 2013
@@ -175,7 +175,6 @@ public class AMContainerHelpers {
     myEnv.putAll(env);
     myEnv.putAll(vertexEnv);
     // TODO TEZ-38 MRChildJVM2.setEnv should become a no-op
-    TezEngineChildJVM.setVMEnv(myEnv, conf, vertexId, appContext);
 
     // Set up the launch command
     List<String> commands = TezEngineChildJVM.getVMCommand(

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
Wed Apr 24 20:51:53 2013
@@ -20,10 +20,8 @@ package org.apache.tez.dag.utils;
 
 import java.net.InetSocketAddress;
 import java.util.List;
-import java.util.Map;
 import java.util.Vector;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.YarnTezDagChild;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -32,7 +30,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.engine.records.TezVertexID;
 
 public class TezEngineChildJVM {
@@ -71,15 +68,6 @@ public class TezEngineChildJVM {
         filter.toString();
   }
   
-  public static void setVMEnv(Map<String, String> environment, TezConfiguration conf,
-      TezVertexID vertexId, AppContext appContext) {
-
-    // FIXME this should be derivable from the container id set by the NM
-    // and not require the AM to set
-    environment.put(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV,
-        String.valueOf(appContext.getApplicationAttemptId().getAttemptId()));
-  }
-
   public static List<String> getVMCommand(
       InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf, 
       TezVertexID vertexId, 
@@ -106,11 +94,10 @@ public class TezEngineChildJVM {
 
     // Add main class and its arguments 
     vargs.add(YarnTezDagChild.class.getName());  // main of Child
+
     // pass TaskAttemptListener's address
     vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
     vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
-    // Set the job id
-    vargs.add(jobID.toString());
 
     // Finally add the containerId.
     vargs.add(String.valueOf(containerId.toString()));

Modified: incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java?rev=1471669&r1=1471668&r2=1471669&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
Wed Apr 24 20:51:53 2013
@@ -549,8 +549,6 @@ public interface MRJobConfig {
   public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
   public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
 
-  public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV";
-
   // This should be the directory where splits file gets localized on the node
   // running ApplicationMaster.
   public static final String JOB_SUBMIT_DIR = "jobSubmitDir";



Mime
View raw message