Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 493B0200B92 for ; Wed, 28 Sep 2016 12:40:58 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 47CD8160AD4; Wed, 28 Sep 2016 10:40:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1B746160AB4 for ; Wed, 28 Sep 2016 12:40:56 +0200 (CEST) Received: (qmail 12404 invoked by uid 500); 28 Sep 2016 10:40:56 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 12389 invoked by uid 99); 28 Sep 2016 10:40:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Sep 2016 10:40:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F27A7DFFD8; Wed, 28 Sep 2016 10:40:55 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: varunsaxena@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena) Date: Wed, 28 Sep 2016 10:40:55 +0000 (UTC) archived-at: Wed, 28 Sep 2016 10:40:58 -0000 Repository: hadoop Updated Branches: refs/heads/trunk bc2656f09 -> 9b0fd01d2 YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b0fd01d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b0fd01d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b0fd01d Branch: refs/heads/trunk Commit: 9b0fd01d2ee002ac4c30c2862e18ca8f1626fa8d Parents: bc2656f Author: Varun Saxena Authored: Wed Sep 28 16:10:10 2016 +0530 Committer: Varun Saxena Committed: Wed Sep 28 16:10:10 2016 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 12 ----------- .../src/main/resources/yarn-default.xml | 13 ------------ .../metrics/ApplicationMetricsConstants.java | 3 +++ .../resourcemanager/amlauncher/AMLauncher.java | 21 -------------------- .../metrics/TimelineServiceV1Publisher.java | 6 ++++++ .../metrics/TimelineServiceV2Publisher.java | 5 +++++ .../metrics/TestSystemMetricsPublisher.java | 18 +++++++++++++++++ .../TestSystemMetricsPublisherForV2.java | 9 +++++++++ 8 files changed, 41 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f3009a1..1421873 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -534,18 +534,6 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; - /** - * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the - * command being executed to the RM log if this property is true. Commands - * may contain sensitive information, such as application or service - * passwords, making logging the commands a security risk. In cases where - * the cluster may be running applications with such commands, this property - * should be set to false. Commands are only logged at the debug level. - */ - public static final String RM_AMLAUNCHER_LOG_COMMAND = - RM_PREFIX + "amlauncher.log.command"; - public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false; - //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index d6c33a2..965b575 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -299,19 +299,6 @@ - - The resource manager will log all commands being executed to the RM log - if this property is true. Commands may contain sensitive information, - such as application or service passwords, making logging the commands a - security risk. In cases where the cluster may be running applications with - such commands this property should be set to false. Commands are only - logged at the debug level. - - yarn.resourcemanager.amlauncher.log.command - false - - - The class to use as the resource scheduler. yarn.resourcemanager.scheduler.class org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index d06b7cb..1774208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -98,4 +98,7 @@ public class ApplicationMetricsConstants { public static final String AM_NODE_LABEL_EXPRESSION = "YARN_AM_NODE_LABEL_EXPRESSION"; + + public static final String AM_CONTAINER_LAUNCH_COMMAND = + "YARN_AM_CONTAINER_LAUNCH_COMMAND"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 181463a..d33360b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -66,7 +66,6 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; /** * The launch of the AM itself. @@ -82,7 +81,6 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; - private final boolean logCommandLine; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -95,9 +93,6 @@ public class AMLauncher implements Runnable { this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); - this.logCommandLine = - conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND, - YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND); } private void connect() throws IOException { @@ -194,22 +189,6 @@ public class AMLauncher implements Runnable { ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); - if (LOG.isDebugEnabled()) { - StringBuilder message = new StringBuilder("Command to launch container "); - - message.append(containerID).append(" : "); - - if (logCommandLine) { - message.append(Joiner.on(",").join(container.getCommands())); - } else { - message.append(" -- Set "); - message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND); - message.append(" to true to reenable command logging"); - } - - LOG.debug(message.toString()); - } - // Populate the current queue name in the environment variable. setupQueueNameEnv(container, applicationMasterContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 7f4ed33..ffbc747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.client.api.TimelineClient; @@ -103,6 +104,11 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { } } + ContainerLaunchContext amContainerSpec = + app.getApplicationSubmissionContext().getAMContainerSpec(); + entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND, + amContainerSpec.getCommands()); + entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index a248199..1485b91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity; @@ -128,6 +129,10 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { app.getCallerContext().getSignature()); } } + ContainerLaunchContext amContainerSpec = + app.getApplicationSubmissionContext().getAMContainerSpec(); + entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND, + amContainerSpec.getCommands()); entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 1e279d5..386932d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.Map; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -118,6 +120,11 @@ public class TestSystemMetricsPublisher { when(asc.getUnmanagedAM()).thenReturn(false); when(asc.getPriority()).thenReturn(Priority.newInstance(1)); when(asc.getNodeLabelExpression()).thenReturn("high-cpu"); + ContainerLaunchContext containerLaunchContext = + mock(ContainerLaunchContext.class); + when(containerLaunchContext.getCommands()) + .thenReturn(Collections.singletonList("java -Xmx1024m")); + when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); metricsPublisher.appUpdated(app, 4L); } else { @@ -197,6 +204,12 @@ public class TestSystemMetricsPublisher { Assert.assertEquals("uers1,user2", entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + + Assert.assertEquals( + app.getApplicationSubmissionContext().getAMContainerSpec() + .getCommands(), + entity.getOtherInfo() + .get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND)); } else { Assert.assertEquals( "", @@ -492,6 +505,11 @@ public class TestSystemMetricsPublisher { when(asc.getUnmanagedAM()).thenReturn(false); when(asc.getPriority()).thenReturn(Priority.newInstance(10)); when(asc.getNodeLabelExpression()).thenReturn("high-cpu"); + ContainerLaunchContext containerLaunchContext = + mock(ContainerLaunchContext.class); + when(containerLaunchContext.getCommands()) + .thenReturn(Collections.singletonList("java -Xmx1024m")); + when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); when(app.getAppNodeLabelExpression()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0fd01d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 3ea4714..13aa806 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -354,6 +355,14 @@ public class TestSystemMetricsPublisherForV2 { mock(ApplicationSubmissionContext.class); when(appSubmissionContext.getPriority()) .thenReturn(Priority.newInstance(0)); + + ContainerLaunchContext containerLaunchContext = + mock(ContainerLaunchContext.class); + when(containerLaunchContext.getCommands()) + .thenReturn(Collections.singletonList("java -Xmx1024m")); + when(appSubmissionContext.getAMContainerSpec()) + .thenReturn(containerLaunchContext); + when(app.getApplicationSubmissionContext()) .thenReturn(appSubmissionContext); return app; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org