Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D680E200B42 for ; Sun, 10 Jul 2016 17:57:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D55DC160A81; Sun, 10 Jul 2016 15:57:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B0CAC160A69 for ; Sun, 10 Jul 2016 17:57:30 +0200 (CEST) Received: (qmail 75491 invoked by uid 500); 10 Jul 2016 15:57:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 72636 invoked by uid 99); 10 Jul 2016 15:57:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 10 Jul 2016 15:57:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 11BC6ED103; Sun, 10 Jul 2016 15:57:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sjlee@apache.org To: common-commits@hadoop.apache.org Date: Sun, 10 Jul 2016 15:57:56 -0000 Message-Id: <52f20e5d5a9444dabf9a1b0771573fa4@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [41/50] [abbrv] hadoop git commit: Cleanup changes during rebase with trunk (Vrushali C) archived-at: Sun, 10 Jul 2016 15:57:32 -0000 Cleanup changes during rebase with trunk (Vrushali C) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d943038 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d943038 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d943038 Branch: refs/heads/trunk Commit: 6d943038f61211594d017f631710e6710ec71372 Parents: 1a22774 Author: Vrushali Authored: Tue Jun 21 16:38:20 2016 -0700 Committer: Sangjin Lee Committed: Sun Jul 10 08:46:04 2016 -0700 ---------------------------------------------------------------------- hadoop-project/pom.xml | 5 ----- hadoop-yarn-project/hadoop-yarn/bin/yarn | 11 +++++----- .../nodemanager/NodeStatusUpdaterImpl.java | 1 - .../containermanager/ContainerManagerImpl.java | 6 ++---- .../application/ApplicationImpl.java | 22 +++++++++----------- .../metrics/TimelineServiceV1Publisher.java | 12 +++++++++++ .../metrics/TimelineServiceV2Publisher.java | 16 +++++++++++++- .../pom.xml | 4 ++-- 8 files changed, 46 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index bb46de0..e6858a2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1133,11 +1133,6 @@ test true - - - jdk.tools - jdk.tools - org.jruby jruby-complete http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index a01f495..dbbaed3 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -48,11 +48,10 @@ function hadoop_usage hadoop_add_subcommand "rmadmin" "admin tools" hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools" hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon" - hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "timelinereader" "run the timeline reader server" + hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "top" "view cluster information" hadoop_add_subcommand "version" "print the version" - hadoop_generate_usage "${MYNAME}" true hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true } @@ -173,6 +172,10 @@ function yarncmd_case hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}" ;; + timelinereader) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' + ;; timelineserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' @@ -184,10 +187,6 @@ function yarncmd_case HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}" fi ;; - timelinereader) - supportdaemonization="true" - CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' - ;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 6e0e760..b37b38e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -81,7 +81,6 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 663a2e1..89e81ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -154,7 +154,6 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import org.apache.hadoop.yarn.util.resource.Resources; public class ContainerManagerImpl extends CompositeService implements ContainerManager { @@ -351,9 +350,8 @@ public class ContainerManagerImpl extends CompositeService implements LOG.info("Recovering application " + appId); //TODO: Recover flow and flow run ID - ApplicationImpl app = new ApplicationImpl( - dispatcher, p.getUser(), null, appId, creds, context, - p.getAppLogAggregationInitedTime()); + ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), appId, + creds, context, p.getAppLogAggregationInitedTime()); context.getApplications().put(appId, app); app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 5a5d30c..531693e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.proto.YarnProtos; @@ -118,10 +117,19 @@ public class ApplicationImpl implements Application { Context context, long recoveredLogInitedTime) { this.dispatcher = dispatcher; this.user = user; - this.flowContext = flowContext; this.appId = appId; this.credentials = credentials; this.aclsManager = context.getApplicationACLsManager(); + Configuration conf = context.getConf(); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (flowContext == null) { + throw new IllegalArgumentException("flow context cannot be null"); + } + this.flowContext = flowContext; + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + context.getNMTimelinePublisher().createTimelineClient(appId); + } + } this.context = context; this.appStateStore = context.getNMStateStore(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -136,16 +144,6 @@ public class ApplicationImpl implements Application { Credentials credentials, Context context) { this(dispatcher, user, flowContext, appId, credentials, context, -1); - Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { - if (flowContext == null) { - throw new IllegalArgumentException("flow context cannot be null"); - } - this.flowContext = flowContext; - if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - context.getNMTimelinePublisher().createTimelineClient(appId); - } - } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index ed8cf06..61e7d65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -181,6 +181,8 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); } + @SuppressWarnings("unchecked") + @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { TimelineEntity entity = createApplicationEntity(app.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); @@ -248,6 +250,10 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { app.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); @@ -301,6 +307,12 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { container.getContainerExitStatus()); eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, container.getContainerState().toString()); + Map entityInfo = new HashMap(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + container.getAllocatedNode().getHost()); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + container.getAllocatedNode().getPort()); + entity.setOtherInfo(entityInfo); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 955570f..a248199 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -196,10 +196,14 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { ApplicationEntity entity = createApplicationEntity(app.getApplicationId()); + TimelineEvent tEvent = new TimelineEvent(); Map entityInfo = new HashMap(); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, - appViewACLs); + (appViewACLs == null) ? "" : appViewACLs); entity.setInfo(entityInfo); + tEvent.setId(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); + tEvent.setTimestamp(updatedTime); + entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); @@ -285,6 +289,10 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { app.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setInfo(eventInfo); entity.addEvent(tEvent); @@ -353,6 +361,12 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { container.getContainerExitStatus()); eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, container.getContainerState().toString()); + Map entityInfo = new HashMap(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + container.getAllocatedNode().getHost()); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + container.getAllocatedNode().getPort()); + entity.setInfo(entityInfo); tEvent.setInfo(eventInfo); entity.addEvent(tEvent); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index a8e5195..5641572 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -22,12 +22,12 @@ hadoop-yarn-server org.apache.hadoop - 3.0.0-SNAPSHOT + 3.0.0-alpha1-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-server-timelineservice-hbase-tests - 3.0.0-SNAPSHOT + 3.0.0-alpha1-SNAPSHOT Apache Hadoop YARN Timeline Service HBase tests --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org