hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject [hadoop] 05/07: YARN-9418. ATSV2 /apps//entities/YARN_CONTAINER rest api does not show metrics. Contributed by Prabhu Joseph.
Date Thu, 05 Sep 2019 10:09:58 GMT
This is an automated email from the ASF dual-hosted git repository.

rohithsharmaks pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 998aa3de2cee8ab40b2297e52a79978c52fbb933
Author: Giovanni Matteo Fumarola <gifuma@apache.org>
AuthorDate: Mon Apr 1 11:06:51 2019 -0700

    YARN-9418. ATSV2 /apps//entities/YARN_CONTAINER rest api does not show metrics. Contributed
by Prabhu Joseph.
---
 .../timelineservice/NMTimelinePublisher.java         |  8 ++------
 .../timelineservice/TestNMTimelinePublisher.java     | 20 +++++++++++++-------
 .../metrics/TimelineServiceV2Publisher.java          |  2 ++
 .../metrics/TestSystemMetricsPublisherForV2.java     |  3 ++-
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index e9bd965..18e3d86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -186,8 +186,6 @@ public class NMTimelinePublisher extends CompositeService {
               Math.round(cpuUsagePercentPerCore));
           entity.addMetric(cpuMetric);
         }
-        entity.setIdPrefix(TimelineServiceHelper.
-            invertLong(container.getContainerStartTime()));
         ApplicationId appId = container.getContainerId().
             getApplicationAttemptId().getApplicationId();
         try {
@@ -253,7 +251,6 @@ public class NMTimelinePublisher extends CompositeService {
       long containerStartTime = container.getContainerStartTime();
       entity.addEvent(tEvent);
       entity.setCreatedTime(containerStartTime);
-      entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime));
       dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
           containerId.getApplicationAttemptId().getApplicationId()));
     }
@@ -281,7 +278,6 @@ public class NMTimelinePublisher extends CompositeService {
       tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
       tEvent.setTimestamp(containerFinishTime);
       entity.addEvent(tEvent);
-      entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime));
 
       dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
           containerId.getApplicationAttemptId().getApplicationId()));
@@ -299,8 +295,6 @@ public class NMTimelinePublisher extends CompositeService {
       tEvent.setId(eventType);
       tEvent.setTimestamp(event.getTimestamp());
       entity.addEvent(tEvent);
-      entity.setIdPrefix(TimelineServiceHelper.
-          invertLong(container.getContainerStartTime()));
 
       ApplicationId appId = container.getContainerId().
           getApplicationAttemptId().getApplicationId();
@@ -336,6 +330,8 @@ public class NMTimelinePublisher extends CompositeService {
       ContainerId containerId) {
     ContainerEntity entity = new ContainerEntity();
     entity.setId(containerId.toString());
+    entity.setIdPrefix(TimelineServiceHelper.invertLong(
+        containerId.getContainerId()));
     Identifier parentIdentifier = new Identifier();
     parentIdentifier
         .setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
index cf9ede0..ae51f85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.After;
@@ -140,6 +141,8 @@ public class TestNMTimelinePublisher {
         entity.getInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
     Assert.assertEquals(exitStatus,
         entity.getInfo().get(ContainerMetricsConstants.EXIT_STATUS_INFO));
+    Assert.assertEquals(TimelineServiceHelper.invertLong(
+        cId.getContainerId()), entity.getIdPrefix());
   }
 
   @Test public void testContainerResourceUsage() {
@@ -148,29 +151,31 @@ public class TestNMTimelinePublisher {
     Container aContainer = mock(Container.class);
     when(aContainer.getContainerId()).thenReturn(ContainerId
         .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L));
+    long idPrefix = TimelineServiceHelper.invertLong(
+        aContainer.getContainerId().getContainerId());
     publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
-    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8);
+    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix);
     timelineClient.reset();
 
     publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F);
-    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1);
+    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix);
     timelineClient.reset();
 
     publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F);
-    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0);
+    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix);
     timelineClient.reset();
 
     publisher.reportContainerResourceUsage(aContainer, 1024L,
         (float) ResourceCalculatorProcessTree.UNAVAILABLE);
     verifyPublishedResourceUsageMetrics(timelineClient, 1024L,
-        ResourceCalculatorProcessTree.UNAVAILABLE);
+        ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix);
   }
 
-  private void verifyPublishedResourceUsageMetrics(
-      DummyTimelineClient timelineClient, long memoryUsage, int cpuUsage) {
+  private void verifyPublishedResourceUsageMetrics(DummyTimelineClient
+      dummyTimelineClient, long memoryUsage, int cpuUsage, long idPrefix) {
     TimelineEntity[] entities = null;
     for (int i = 0; i < 10; i++) {
-      entities = timelineClient.getLastPublishedEntities();
+      entities = dummyTimelineClient.getLastPublishedEntities();
       if (entities != null) {
         break;
       }
@@ -188,6 +193,7 @@ public class TestNMTimelinePublisher {
     assertNotNull("entities are expected to be published", entities);
     assertEquals("Expected number of metrics notpublished",
         numberOfResourceMetrics, entities[0].getMetrics().size());
+    assertEquals(idPrefix, entities[0].getIdPrefix());
     Iterator<TimelineMetric> metrics = entities[0].getMetrics().iterator();
     while (metrics.hasNext()) {
       TimelineMetric metric = metrics.next();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 11db6b1..35093e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -441,6 +441,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher
{
       ContainerId containerId) {
     ContainerEntity entity = new ContainerEntity();
     entity.setId(containerId.toString());
+    entity.setIdPrefix(TimelineServiceHelper.invertLong(
+        containerId.getContainerId()));
     entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT
         .name(), containerId.getApplicationAttemptId().toString()));
     return entity;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index ca26f58..5a19410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -293,7 +293,8 @@ public class TestSystemMetricsPublisherForV2 {
     File appFile = new File(outputDirApp, timelineServiceFileName);
     Assert.assertTrue(appFile.exists());
     verifyEntity(appFile, 2,
-        ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0);
+        ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0,
+        TimelineServiceHelper.invertLong(containerId.getContainerId()));
   }
 
   @Test(timeout = 10000)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message