tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2179. Timeline relatedentries missing cause exaggerated warning. (Chang Li via hitesh)
Date Thu, 12 Mar 2015 14:47:51 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.6 559099ea6 -> b649dda68


TEZ-2179. Timeline relatedentries missing cause exaggerated warning. (Chang Li via hitesh)

(cherry picked from commit 19df277868e2775bb92ac80c6c7fd147724f9268)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b649dda6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b649dda6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b649dda6

Branch: refs/heads/branch-0.6
Commit: b649dda68e5e5982276c59bf8df405018d5f6853
Parents: 559099e
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Mar 12 07:46:43 2015 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Mar 12 07:48:18 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../ats/HistoryEventTimelineConversion.java     | 32 +++++-------
 .../ats/TestHistoryEventTimelineConversion.java | 53 ++++++++------------
 3 files changed, 35 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/b649dda6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ea4b0d5..aebdb36 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2179. Timeline relatedentries missing cause exaggerated warning.
   TEZ-2168. Fix application dependencies on mutually exclusive artifacts: tez-yarn-timeline-history
     and tez-yarn-timeline-history-with-acls.
   TEZ-2190. TestOrderedWordCount fails when generateSplitsInClient set to true.

http://git-wip-us.apache.org/repos/asf/tez/blob/b649dda6/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 8600776..2e59d92 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -135,14 +135,13 @@ public class HistoryEventTimelineConversion {
         + event.getApplicationId().toString());
     atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION.name());
 
-    atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
-        event.getApplicationId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
-
     atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
 
     atsEntity.addOtherInfo(ATSConstants.CONFIG,
         DAGUtils.convertConfigurationToATSMap(event.getConf()));
+    atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
+            event.getApplicationId().toString());
+    atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
 
     atsEntity.setStartTime(event.getLaunchTime());
 
@@ -160,12 +159,6 @@ public class HistoryEventTimelineConversion {
         + event.getApplicationAttemptId().toString());
     atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
 
-    atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
-        event.getApplicationAttemptId().getApplicationId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
-        event.getApplicationAttemptId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
-
     atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
     atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
         event.getApplicationAttemptId().getApplicationId().toString());
@@ -178,6 +171,11 @@ public class HistoryEventTimelineConversion {
     atsEntity.addEvent(launchEvt);
 
     atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime());
+    atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
+        event.getApplicationAttemptId().getApplicationId().toString());
+    atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
+        event.getApplicationAttemptId().toString());
+    atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
 
     return atsEntity;
   }
@@ -208,12 +206,12 @@ public class HistoryEventTimelineConversion {
 
     atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
         "tez_" + event.getApplicationAttemptId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID,
-        event.getContainerId().toString());
 
     atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
         event.getApplicationAttemptId().getApplicationId().toString());
 
+    atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID,
+            event.getContainerId().toString());
     atsEntity.setStartTime(event.getLaunchTime());
 
     TimelineEvent launchEvt = new TimelineEvent();
@@ -341,11 +339,6 @@ public class HistoryEventTimelineConversion {
         "tez_" + event.getApplicationAttemptId().getApplicationId().toString());
     atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
         "tez_" + event.getApplicationAttemptId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
-        event.getApplicationAttemptId().getApplicationId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
-        event.getApplicationAttemptId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
 
     TimelineEvent submitEvt = new TimelineEvent();
     submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name());
@@ -367,6 +360,9 @@ public class HistoryEventTimelineConversion {
     }
     atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
         event.getApplicationAttemptId().getApplicationId().toString());
+    atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
+            event.getApplicationAttemptId().toString());
+    atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
 
     return atsEntity;
   }
@@ -412,8 +408,6 @@ public class HistoryEventTimelineConversion {
 
     atsEntity.setStartTime(event.getStartTime());
 
-    atsEntity.addRelatedEntity(ATSConstants.NODE_ID, event.getNodeId().toString());
-    atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID, event.getContainerId().toString());
     atsEntity.addRelatedEntity(EntityTypes.TEZ_TASK_ID.name(),
         event.getTaskAttemptID().getTaskID().toString());
 

http://git-wip-us.apache.org/repos/asf/tez/blob/b649dda6/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index 6d713c5..a2b0f89 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -222,18 +222,17 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(EntityTypes.TEZ_APPLICATION.name(), timelineEntity.getEntityType());
     Assert.assertEquals("tez_" + applicationId.toString(), timelineEntity.getEntityId());
 
-    Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
-    Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user));
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains(
-            applicationId.toString()));
+    Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());
 
     Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
     Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
 
-    Assert.assertEquals(2, timelineEntity.getOtherInfo().size());
+    Assert.assertEquals(4, timelineEntity.getOtherInfo().size());
     Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
     Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION));
+    Assert.assertEquals(user, timelineEntity.getOtherInfo().get(ATSConstants.USER));
+    Assert.assertEquals(applicationId.toString(),
+           timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
 
     Map<String, String> config =
         (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
@@ -263,12 +262,7 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), timelineEntity.getEntityType());
 
     final Map<String, Set<String>> relatedEntities = timelineEntity.getRelatedEntities();
-    Assert.assertEquals(3, relatedEntities.size());
-    Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ID)
-        .contains(applicationId.toString()));
-    Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ATTEMPT_ID)
-        .contains(applicationAttemptId.toString()));
-    Assert.assertTrue(relatedEntities.get(ATSConstants.USER).contains(user));
+    Assert.assertEquals(0, relatedEntities.size());
 
     final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
     Assert.assertEquals(2, primaryFilters.size());
@@ -284,8 +278,11 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(launchTime, evt.getTimestamp());
 
     final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
-    Assert.assertEquals(1, otherInfo.size());
+    Assert.assertEquals(4, otherInfo.size());
     Assert.assertEquals(submitTime, otherInfo.get(ATSConstants.APP_SUBMIT_TIME));
+    Assert.assertEquals(applicationId.toString(), otherInfo.get(ATSConstants.APPLICATION_ID));
+    Assert.assertEquals(applicationAttemptId.toString(), otherInfo.get(ATSConstants.APPLICATION_ATTEMPT_ID));
+    Assert.assertEquals(user, otherInfo.get(ATSConstants.USER));
   }
 
   @Test(timeout = 5000)
@@ -298,9 +295,7 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(EntityTypes.TEZ_CONTAINER_ID.name(), timelineEntity.getEntityType());
     Assert.assertEquals("tez_" + containerId.toString(), timelineEntity.getEntityId());
 
-    Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
-    Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains(
-        containerId.toString()));
+    Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
     Assert.assertTrue(
         timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains(
             "tez_" + applicationAttemptId.toString()));
@@ -309,6 +304,8 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.APPLICATION_ID).contains(
         applicationAttemptId.getApplicationId().toString()));
 
+    Assert.assertEquals(containerId.toString(),timelineEntity.getOtherInfo().get(ATSConstants.CONTAINER_ID));
+
     Assert.assertEquals(launchTime, timelineEntity.getStartTime().longValue());
 
     Assert.assertEquals(1, timelineEntity.getEvents().size());
@@ -389,21 +386,13 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
     Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
 
-    Assert.assertEquals(5, timelineEntity.getRelatedEntities().size());
+    Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
     Assert.assertTrue(
         timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION.name()).contains(
             "tez_" + applicationId.toString()));
     Assert.assertTrue(
         timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains(
             "tez_" + applicationAttemptId.toString()));
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ATTEMPT_ID).contains(
-            applicationAttemptId.toString()));
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains(
-            applicationAttemptId.getApplicationId().toString()));
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user));
 
     Assert.assertEquals(1, timelineEntity.getEvents().size());
     TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
@@ -426,7 +415,12 @@ public class TestHistoryEventTimelineConversion {
     Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.DAG_PLAN));
     Assert.assertEquals(applicationId.toString(),
         timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
-
+    Assert.assertEquals(applicationAttemptId.toString(),
+        timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ATTEMPT_ID));
+    Assert.assertEquals(applicationAttemptId.getApplicationId().toString(),
+        timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
+    Assert.assertEquals(user,
+           timelineEntity.getOtherInfo().get(ATSConstants.USER));
   }
 
   @Test(timeout = 5000)
@@ -721,12 +715,7 @@ public class TestHistoryEventTimelineConversion {
 
     Assert.assertEquals(startTime, timelineEntity.getStartTime().longValue());
 
-    Assert.assertEquals(3, timelineEntity.getRelatedEntities().size());
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.NODE_ID).contains(nodeId.toString()));
-    Assert.assertTrue(
-        timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains(
-            containerId.toString()));
+    Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
     Assert.assertTrue(
         timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_TASK_ID.name()).contains(
             tezTaskID.toString()));


Mime
View raw message