hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-4747. AHS error 500 due to NPE when container start event is missing. Contributed by Varun Saxena (cherry picked from commit b2ed6ae73197990a950ce71ece80c0f23221c384)
Date Fri, 06 May 2016 23:00:46 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b68e6b1d6 -> 3895058a6


YARN-4747. AHS error 500 due to NPE when container start event is missing. Contributed by
Varun Saxena
(cherry picked from commit b2ed6ae73197990a950ce71ece80c0f23221c384)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3895058a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3895058a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3895058a

Branch: refs/heads/branch-2
Commit: 3895058a675cfcda8d6fa7c7a969251fa2db2a66
Parents: b68e6b1
Author: Jason Lowe <jlowe@apache.org>
Authored: Fri May 6 22:58:17 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Fri May 6 23:00:25 2016 +0000

----------------------------------------------------------------------
 ...pplicationHistoryManagerOnTimelineStore.java | 21 ++++++++------
 .../metrics/ContainerFinishedEvent.java         |  9 +++++-
 .../metrics/SystemMetricsPublisher.java         |  8 +++++-
 .../metrics/TestSystemMetricsPublisher.java     | 30 ++++++++++++++++++++
 4 files changed, 57 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index fd11aea..19afb25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -587,19 +587,22 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         }
       }
     }
-    NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
     ContainerId containerId =
         ConverterUtils.toContainerId(entity.getEntityId());
-    String logUrl = WebAppUtils.getAggregatedLogURL(
-        serverHttpAddress,
-        allocatedNode.toString(),
-        containerId.toString(),
-        containerId.toString(),
-        user);
+    String logUrl = null;
+    NodeId allocatedNode = null;
+    if (allocatedHost != null) {
+      allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
+      logUrl = WebAppUtils.getAggregatedLogURL(
+          serverHttpAddress,
+          allocatedNode.toString(),
+          containerId.toString(),
+          containerId.toString(),
+          user);
+    }
     return ContainerReport.newInstance(
         ConverterUtils.toContainerId(entity.getEntityId()),
-        Resource.newInstance(allocatedMem, allocatedVcore),
-        NodeId.newInstance(allocatedHost, allocatedPort),
+        Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode,
         Priority.newInstance(allocatedPriority),
         createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
         nodeHttpAddress);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java
index aafd760..ca4d311 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.NodeId;
 
 public class ContainerFinishedEvent extends SystemMetricsEvent {
 
@@ -27,17 +28,20 @@ public class ContainerFinishedEvent extends SystemMetricsEvent {
   private String diagnosticsInfo;
   private int containerExitStatus;
   private ContainerState state;
+  private NodeId allocatedNode;
 
   public ContainerFinishedEvent(
       ContainerId containerId,
       String diagnosticsInfo,
       int containerExitStatus,
       ContainerState state,
-      long finishedTime) {
+      long finishedTime,
+      NodeId allocatedNode) {
     super(SystemMetricsEventType.CONTAINER_FINISHED, finishedTime);
     this.containerId = containerId;
     this.diagnosticsInfo = diagnosticsInfo;
     this.containerExitStatus = containerExitStatus;
+    this.allocatedNode = allocatedNode;
     this.state = state;
   }
 
@@ -62,4 +66,7 @@ public class ContainerFinishedEvent extends SystemMetricsEvent {
     return state;
   }
 
+  public NodeId getAllocatedNode() {
+    return allocatedNode;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 84a3b19..cba8790 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -218,7 +218,7 @@ public class SystemMetricsPublisher extends CompositeService {
               container.getDiagnosticsInfo(),
               container.getContainerExitStatus(),
               container.getContainerState(),
-              finishedTime));
+              finishedTime, container.getAllocatedNode()));
     }
   }
 
@@ -479,6 +479,12 @@ public class SystemMetricsPublisher extends CompositeService {
         event.getContainerExitStatus());
     eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
         event.getContainerState().toString());
+    Map<String, Object> entityInfo = new HashMap<String, Object>();
+    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+        event.getAllocatedNode().getHost());
+    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
+        event.getAllocatedNode().getPort());
+    entity.setOtherInfo(entityInfo);
     tEvent.setEventInfo(eventInfo);
     entity.addEvent(tEvent);
     putEntity(entity);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index f2b02bc..0738a2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -345,6 +345,36 @@ public class TestSystemMetricsPublisher {
   }
 
   @Test(timeout = 10000)
+  public void testPublishHostPortInfoOnContainerFinished() throws Exception {
+    ContainerId containerId =
+        ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+            ApplicationId.newInstance(0, 1), 1), 1);
+    RMContainer container = createRMContainer(containerId);
+    metricsPublisher.containerFinished(container, container.getFinishTime());
+    TimelineEntity entity = null;
+    do {
+      entity =
+          store.getEntity(containerId.toString(),
+              ContainerMetricsConstants.ENTITY_TYPE,
+              EnumSet.allOf(Field.class));
+    } while (entity == null || entity.getEvents().size() < 1);
+    Assert.assertNotNull(entity.getOtherInfo());
+    Assert.assertEquals(2, entity.getOtherInfo().size());
+    Assert.assertNotNull(entity.getOtherInfo().get(
+        ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO));
+    Assert.assertNotNull(entity.getOtherInfo().get(
+        ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO));
+    Assert.assertEquals(
+        container.getAllocatedNode().getHost(),
+        entity.getOtherInfo().get(
+            ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO));
+    Assert.assertEquals(
+        container.getAllocatedNode().getPort(),
+        entity.getOtherInfo().get(
+            ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO));
+  }
+
+  @Test(timeout = 10000)
   public void testPublishContainerMetrics() throws Exception {
     ContainerId containerId =
         ContainerId.newContainerId(ApplicationAttemptId.newInstance(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message