Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 418EB10CBA for ; Tue, 24 Feb 2015 04:53:43 +0000 (UTC) Received: (qmail 49794 invoked by uid 500); 24 Feb 2015 04:53:43 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 49758 invoked by uid 500); 24 Feb 2015 04:53:43 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 49749 invoked by uid 99); 24 Feb 2015 04:53:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Feb 2015 04:53:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0E683E03E8; Tue, 24 Feb 2015 04:53:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.apache.org Message-Id: <7221a246dafc4ee5837ad66ac9920d85@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. (sseth) Date: Tue, 24 Feb 2015 04:53:43 +0000 (UTC) Repository: tez Updated Branches: refs/heads/TEZ-2003 08cd3c2b9 -> d15b812e5 TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d15b812e Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d15b812e Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d15b812e Branch: refs/heads/TEZ-2003 Commit: d15b812e5707ae3d96a632cad1e8e5ff60e32286 Parents: 08cd3c2 Author: Siddharth Seth Authored: Mon Feb 23 20:53:24 2015 -0800 Committer: Siddharth Seth Committed: Mon Feb 23 20:53:24 2015 -0800 ---------------------------------------------------------------------- TEZ-2003-CHANGES.txt | 1 + .../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +- .../java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 4 ++++ .../org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java | 1 + .../dag/app/launcher/TezTestServiceContainerLauncher.java | 8 +++++--- 5 files changed, 12 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/d15b812e/TEZ-2003-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt index 4b0a12b..4377f57 100644 --- a/TEZ-2003-CHANGES.txt +++ b/TEZ-2003-CHANGES.txt @@ -7,5 +7,6 @@ ALL CHANGES: TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode) TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM. TEZ-2131. Add additional tests for tasks running in the AM. + TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. INCOMPATIBLE CHANGES: http://git-wip-us.apache.org/repos/asf/tez/blob/d15b812e/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index b031d89..5588dc3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -1404,7 +1404,7 @@ public class DAGAppMaster extends AbstractService { @Override public Integer getContainerLauncherIdentifier(String name) { - return taskCommunicators.get(name); + return containerLaunchers.get(name); } @Override http://git-wip-us.apache.org/repos/asf/tez/blob/d15b812e/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 023d6c6..1d599c3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -842,6 +842,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, .get(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, tezDefaultComponentName); String containerLauncherName = vertexConf .get(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, tezDefaultComponentName); + + LOG.info("Vertex: " + logIdentifier + " configured with TaskScheduler=" + taskSchedulerName + + ", ContainerLauncher=" + containerLauncherName + ", TaskComm=" + taskCommName); + taskSchedulerIdentifier = appContext.getTaskScheduerIdentifier(taskSchedulerName); taskCommunicatorIdentifier = appContext.getTaskCommunicatorIdentifier(taskCommName); containerLauncherIdentifier = appContext.getContainerLauncherIdentifier(containerLauncherName); http://git-wip-us.apache.org/repos/asf/tez/blob/d15b812e/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java index 02952d0..1ffa3fa 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java @@ -174,6 +174,7 @@ public class TaskSchedulerEventHandler extends AbstractService for (int i = 0 ; i < schedulerClasses.length ; i++) { // Copy over the rest. this.taskSchedulerClasses[i] = schedulerClasses[i]; } + this.taskSchedulerClasses[foundYarnTaskSchedulerIndex] = TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT; } else { this.taskSchedulerClasses = schedulerClasses; } http://git-wip-us.apache.org/repos/asf/tez/blob/d15b812e/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java ---------------------------------------------------------------------- diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java index 27356bc..cb6308c 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.Clock; import org.apache.tez.dag.app.AppContext; @@ -54,6 +55,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements private final int servicePort; private final TezTestServiceCommunicator communicator; private final Clock clock; + private final ApplicationAttemptId appAttemptId; // Configuration passed in here to set up final parameters @@ -70,6 +72,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements this.communicator = new TezTestServiceCommunicator(numThreads); this.context = appContext; this.tokenIdentifier = context.getApplicationID().toString(); + this.appAttemptId = appContext.getApplicationAttemptId(); this.tal = tal; } @@ -128,9 +131,8 @@ public class TezTestServiceContainerLauncher extends AbstractService implements RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder(); InetSocketAddress address = tal.getTaskCommunicator(event.getTaskCommId()).getAddress(); builder.setAmHost(address.getHostName()).setAmPort(address.getPort()); - builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId()); - builder.setApplicationIdString( - event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString()); + builder.setAppAttemptNumber(appAttemptId.getAttemptId()); + builder.setApplicationIdString(appAttemptId.getApplicationId().toString()); builder.setTokenIdentifier(tokenIdentifier); builder.setContainerIdString(event.getContainer().getId().toString()); builder.setCredentialsBinary(