tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-3609. Improve ATSv15 performance for DAG entities read calls. Contributed by Harish Jaiprakash.
Date Wed, 08 Feb 2017 06:54:32 GMT
Repository: tez
Updated Branches:
  refs/heads/master 63ae97d5f -> 11815a7cc


TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
Contributed by Harish Jaiprakash.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/11815a7c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/11815a7c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/11815a7c

Branch: refs/heads/master
Commit: 11815a7ccd80ac3cb052f34a420d44de9f083542
Parents: 63ae97d
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Feb 7 22:54:08 2017 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Feb 7 22:54:08 2017 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../logging/ats/TimelineCachePluginImpl.java    |  1 -
 .../ats/TestTimelineCachePluginImpl.java        | 41 ++++++++++----------
 3 files changed, 23 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a7cc0ce..0949339 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
   TEZ-3244. Allow overlap of input and output memory when they are not concurrent
   TEZ-3581. Add different logger to enable suppressing logs for specific lines.
   TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
@@ -198,6 +199,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
   TEZ-3244. Allow overlap of input and output memory when they are not concurrent
   TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
   TEZ-3462. Task attempt failure during container shutdown loses useful container diagnostics

http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
index b4217a1..8269714 100644
--- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
+++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
@@ -72,7 +72,6 @@ public class TimelineCachePluginImpl extends TimelineEntityGroupPlugin implement
   private Set<TimelineEntityGroupId> createTimelineEntityGroupIds(TezDAGID dagId) {
     ApplicationId appId = dagId.getApplicationId();
     HashSet<TimelineEntityGroupId> groupIds = Sets.newHashSet(
-        TimelineEntityGroupId.newInstance(appId, appId.toString()),
         TimelineEntityGroupId.newInstance(appId, dagId.toString()));
     for (int numGroupsPerDag : allNumGroupsPerDag) {
       groupIds.add(TimelineEntityGroupId.newInstance(appId, dagId.getGroupId(numGroupsPerDag)));

http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
index 6f819ba..3d1af63 100644
--- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
+++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
@@ -124,12 +124,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(3, groupIds.size());
+      Assert.assertEquals(2, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId1, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -143,12 +143,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(2, groupIds.size());
+      Assert.assertEquals(1, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId1, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -162,12 +162,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(2, groupIds.size());
+      Assert.assertEquals(1, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId1, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -181,12 +181,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(3, groupIds.size());
+      Assert.assertEquals(2, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId1, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -200,12 +200,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(4, groupIds.size());
+      Assert.assertEquals(3, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId2, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -219,12 +219,13 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(5, groupIds.size());
+      Assert.assertEquals(4, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId2, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(
+            getGroupIds(dagID2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -238,12 +239,12 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(3, groupIds.size());
+      Assert.assertEquals(2, groupIds.size());
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         Assert.assertEquals(appId2, groupId.getApplicationId());
-        Assert.assertTrue(getGroupIds(dagID2, appId2, 100).contains(groupId.getTimelineEntityGroupId()));
+        Assert.assertTrue(getGroupIds(dagID2, 100).contains(groupId.getTimelineEntityGroupId()));
       }
     }
   }
@@ -261,21 +262,21 @@ public class TestTimelineCachePluginImpl {
         Assert.assertNull(groupIds);
         continue;
       }
-      Assert.assertEquals(6, groupIds.size());
+      Assert.assertEquals(4, groupIds.size());
       int found = 0;
       Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
       while (iter.hasNext()) {
         TimelineEntityGroupId groupId = iter.next();
         if (groupId.getApplicationId().equals(appId1)) {
           String entityGroupId = groupId.getTimelineEntityGroupId();
-          if (getGroupIds(dagID1, appId1, 100).contains(entityGroupId)) {
+          if (getGroupIds(dagID1, 100).contains(entityGroupId)) {
             ++found;
           } else {
             Assert.fail("Unexpected group id: " + entityGroupId);
           }
         } else if (groupId.getApplicationId().equals(appId2)) {
           String entityGroupId = groupId.getTimelineEntityGroupId();
-          if (getGroupIds(dagID2, appId2, 100).contains(entityGroupId)) {
+          if (getGroupIds(dagID2, 100).contains(entityGroupId)) {
             ++found;
           } else {
             Assert.fail("Unexpected group id: " + entityGroupId);
@@ -284,7 +285,7 @@ public class TestTimelineCachePluginImpl {
           Assert.fail("Unexpected appId: " + groupId.getApplicationId());
         }
       }
-      Assert.assertEquals("All groupIds not returned", 6, found);
+      Assert.assertEquals("All groupIds not returned", 4, found);
     }
   }
 
@@ -374,8 +375,8 @@ public class TestTimelineCachePluginImpl {
     Assert.assertEquals("All groupIds not returned", 1, found);
   }
 
-  private Set<String> getGroupIds(TezDAGID dagId, ApplicationId appId, int ... allNumDagsPerGroup)
{
-    HashSet<String> groupIds = Sets.newHashSet(dagId.toString(), appId.toString());
+  private Set<String> getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) {
+    HashSet<String> groupIds = Sets.newHashSet(dagId.toString());
     for (int numDagsPerGroup : allNumDagsPerGroup) {
       groupIds.add(dagId.getGroupId(numDagsPerGroup));
     }


Mime
View raw message