Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0812810469 for ; Wed, 24 Apr 2013 20:52:20 +0000 (UTC) Received: (qmail 26795 invoked by uid 500); 24 Apr 2013 20:52:20 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 26770 invoked by uid 500); 24 Apr 2013 20:52:19 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 26754 invoked by uid 99); 24 Apr 2013 20:52:19 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Apr 2013 20:52:19 +0000 X-ASF-Spam-Status: No, hits=-1998.0 required=5.0 tests=ALL_TRUSTED,FB_GET_MEDS X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Apr 2013 20:52:15 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 731CE23889FA; Wed, 24 Apr 2013 20:51:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1471669 - in /incubator/tez/branches/TEZ-1: tez-dag-api/src/main/java/org/apache/tez/dag/api/ tez-dag/src/main/java/org/apache/hadoop/mapred/ tez-dag/src/main/java/org/apache/tez/dag/app/ tez-dag/src/main/java/org/apache/tez/dag/app/rm/con... Date: Wed, 24 Apr 2013 20:51:54 -0000 To: commits@tez.incubator.apache.org From: hitesh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130424205154.731CE23889FA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: hitesh Date: Wed Apr 24 20:51:53 2013 New Revision: 1471669 URL: http://svn.apache.org/r1471669 Log: TEZ-77. Fix build break due to changes in YARN related to ApplicationConstants.Environment. (hitesh) Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java (original) +++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java Wed Apr 24 20:51:53 2013 @@ -91,9 +91,6 @@ public class TezConfiguration extends Co TEZ_HOME_ENV + "/lib/*" }; - public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV"; - - public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml"; } Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java (original) +++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java Wed Apr 24 20:51:53 2013 @@ -60,6 +60,7 @@ import org.apache.hadoop.util.StringUtil import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -105,9 +106,9 @@ public class YarnTezDagChild { final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port); - final ApplicationId appID = ConverterUtils.toApplicationId(args[2]); - - final ContainerId containerId = ConverterUtils.toContainerId(args[3]); + final ContainerId containerId = ConverterUtils.toContainerId(args[2]); + final ApplicationId appID = + containerId.getApplicationAttemptId().getApplicationId(); // FIXME fix initialize metrics in child runner DefaultMetricsSystem.initialize("VertexTask"); @@ -172,7 +173,8 @@ public class YarnTezDagChild { taskAttemptId = taskContext.getTaskAttemptId(); final Task t = createAndConfigureTezTask(taskContext, umbilical, - credentials, jt); + credentials, jt, + containerId.getApplicationAttemptId().getAttemptId()); task = (MRTask) t.getProcessor(); final JobConf job = task.getConf(); @@ -251,7 +253,7 @@ public class YarnTezDagChild { */ private static void configureLocalDirs(MRTask task, JobConf job) throws IOException { String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIRS.name())); job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs); LOG.info(TezJobConfig.LOCAL_DIR + " for child: " + job.get(TezJobConfig.LOCAL_DIR)); @@ -289,15 +291,12 @@ public class YarnTezDagChild { } private static JobConf configureTask(MRTask task, Credentials credentials, - Token jt) throws IOException, InterruptedException { + Token jt, int appAttemptId) + throws IOException, InterruptedException { JobConf job = task.getConf(); - String appAttemptIdEnv = System - .getenv(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV); - LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv); // Set it in conf, so as to be able to be used the the OutputCommitter. - job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer - .parseInt(appAttemptIdEnv)); + job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId); // set tcp nodelay job.setBoolean("ipc.client.tcpnodelay", true); @@ -406,8 +405,9 @@ public class YarnTezDagChild { private static Task createAndConfigureTezTask( TezEngineTaskContext taskContext, - TezTaskUmbilicalProtocol master, - Credentials credentials, Token jt) + TezTaskUmbilicalProtocol master, + Credentials credentials, Token jt, + int appAttemptId) throws IOException, InterruptedException { Configuration jConf = new JobConf(MRJobConfig.JOB_CONF_FILE); Configuration conf; @@ -466,7 +466,7 @@ public class YarnTezDagChild { t.initialize(job, master); MRTask task = (MRTask)t.getProcessor(); - configureTask(task, credentials, jt); + configureTask(task, credentials, jt, appAttemptId); return t; } Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java (original) +++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java Wed Apr 24 20:51:53 2013 @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.SystemCloc import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1239,20 +1240,14 @@ public class DAGAppMaster extends Compos // TODO: Deprecated keys? //DeprecatedKeys.init(); String containerIdStr = - System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); - String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV); - String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV); + System.getenv(Environment.CONTAINER_ID.name()); + String nodeHostString = System.getenv(Environment.NM_HOST.name()); + String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = - System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); + System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); - validateInputParam(containerIdStr, - ApplicationConstants.AM_CONTAINER_ID_ENV); - validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV); - validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV); - validateInputParam(nodeHttpPortString, - ApplicationConstants.NM_HTTP_PORT_ENV); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java (original) +++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java Wed Apr 24 20:51:53 2013 @@ -175,7 +175,6 @@ public class AMContainerHelpers { myEnv.putAll(env); myEnv.putAll(vertexEnv); // TODO TEZ-38 MRChildJVM2.setEnv should become a no-op - TezEngineChildJVM.setVMEnv(myEnv, conf, vertexId, appContext); // Set up the launch command List commands = TezEngineChildJVM.getVMCommand( Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java (original) +++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java Wed Apr 24 20:51:53 2013 @@ -20,10 +20,8 @@ package org.apache.tez.dag.utils; import java.net.InetSocketAddress; import java.util.List; -import java.util.Map; import java.util.Vector; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.YarnTezDagChild; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -32,7 +30,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.dag.api.TezConfiguration; -import org.apache.tez.dag.app.AppContext; import org.apache.tez.engine.records.TezVertexID; public class TezEngineChildJVM { @@ -71,15 +68,6 @@ public class TezEngineChildJVM { filter.toString(); } - public static void setVMEnv(Map environment, TezConfiguration conf, - TezVertexID vertexId, AppContext appContext) { - - // FIXME this should be derivable from the container id set by the NM - // and not require the AM to set - environment.put(TezConfiguration.APPLICATION_ATTEMPT_ID_ENV, - String.valueOf(appContext.getApplicationAttemptId().getAttemptId())); - } - public static List getVMCommand( InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf, TezVertexID vertexId, @@ -106,11 +94,10 @@ public class TezEngineChildJVM { // Add main class and its arguments vargs.add(YarnTezDagChild.class.getName()); // main of Child + // pass TaskAttemptListener's address vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); - // Set the job id - vargs.add(jobID.toString()); // Finally add the containerId. vargs.add(String.valueOf(containerId.toString())); Modified: incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java?rev=1471669&r1=1471668&r2=1471669&view=diff ============================================================================== --- incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java (original) +++ incubator/tez/branches/TEZ-1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java Wed Apr 24 20:51:53 2013 @@ -549,8 +549,6 @@ public interface MRJobConfig { public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; - public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV"; - // This should be the directory where splits file gets localized on the node // running ApplicationMaster. public static final String JOB_SUBMIT_DIR = "jobSubmitDir";