Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 96D12200C15 for ; Wed, 8 Feb 2017 07:54:34 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 954AB160B5A; Wed, 8 Feb 2017 06:54:34 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BB9BC160B4E for ; Wed, 8 Feb 2017 07:54:33 +0100 (CET) Received: (qmail 21416 invoked by uid 500); 8 Feb 2017 06:54:32 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 21407 invoked by uid 99); 8 Feb 2017 06:54:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Feb 2017 06:54:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AFF9EDFC47; Wed, 8 Feb 2017 06:54:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.apache.org Message-Id: <4a8c74485d7a4b75b1af198c534a78a2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-3609. Improve ATSv15 performance for DAG entities read calls. Contributed by Harish Jaiprakash. Date: Wed, 8 Feb 2017 06:54:32 +0000 (UTC) archived-at: Wed, 08 Feb 2017 06:54:34 -0000 Repository: tez Updated Branches: refs/heads/master 63ae97d5f -> 11815a7cc TEZ-3609. Improve ATSv15 performance for DAG entities read calls. Contributed by Harish Jaiprakash. Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/11815a7c Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/11815a7c Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/11815a7c Branch: refs/heads/master Commit: 11815a7ccd80ac3cb052f34a420d44de9f083542 Parents: 63ae97d Author: Siddharth Seth Authored: Tue Feb 7 22:54:08 2017 -0800 Committer: Siddharth Seth Committed: Tue Feb 7 22:54:08 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../logging/ats/TimelineCachePluginImpl.java | 1 - .../ats/TestTimelineCachePluginImpl.java | 41 ++++++++++---------- 3 files changed, 23 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index a7cc0ce..0949339 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent TEZ-3581. Add different logger to enable suppressing logs for specific lines. TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels @@ -198,6 +199,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3609. Improve ATSv15 performance for DAG entities read calls. TEZ-3244. Allow overlap of input and output memory when they are not concurrent TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels TEZ-3462. Task attempt failure during container shutdown loses useful container diagnostics http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java index b4217a1..8269714 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java @@ -72,7 +72,6 @@ public class TimelineCachePluginImpl extends TimelineEntityGroupPlugin implement private Set createTimelineEntityGroupIds(TezDAGID dagId) { ApplicationId appId = dagId.getApplicationId(); HashSet groupIds = Sets.newHashSet( - TimelineEntityGroupId.newInstance(appId, appId.toString()), TimelineEntityGroupId.newInstance(appId, dagId.toString())); for (int numGroupsPerDag : allNumGroupsPerDag) { groupIds.add(TimelineEntityGroupId.newInstance(appId, dagId.getGroupId(numGroupsPerDag))); http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java index 6f819ba..3d1af63 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java @@ -124,12 +124,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -143,12 +143,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(2, groupIds.size()); + Assert.assertEquals(1, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId())); } } } @@ -162,12 +162,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(2, groupIds.size()); + Assert.assertEquals(1, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId())); } } } @@ -181,12 +181,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId1, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -200,12 +200,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(4, groupIds.size()); + Assert.assertEquals(3, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 50).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID2, 100, 50).contains(groupId.getTimelineEntityGroupId())); } } } @@ -219,12 +219,13 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(5, groupIds.size()); + Assert.assertEquals(4, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue( + getGroupIds(dagID2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId())); } } } @@ -238,12 +239,12 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(3, groupIds.size()); + Assert.assertEquals(2, groupIds.size()); Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); Assert.assertEquals(appId2, groupId.getApplicationId()); - Assert.assertTrue(getGroupIds(dagID2, appId2, 100).contains(groupId.getTimelineEntityGroupId())); + Assert.assertTrue(getGroupIds(dagID2, 100).contains(groupId.getTimelineEntityGroupId())); } } } @@ -261,21 +262,21 @@ public class TestTimelineCachePluginImpl { Assert.assertNull(groupIds); continue; } - Assert.assertEquals(6, groupIds.size()); + Assert.assertEquals(4, groupIds.size()); int found = 0; Iterator iter = groupIds.iterator(); while (iter.hasNext()) { TimelineEntityGroupId groupId = iter.next(); if (groupId.getApplicationId().equals(appId1)) { String entityGroupId = groupId.getTimelineEntityGroupId(); - if (getGroupIds(dagID1, appId1, 100).contains(entityGroupId)) { + if (getGroupIds(dagID1, 100).contains(entityGroupId)) { ++found; } else { Assert.fail("Unexpected group id: " + entityGroupId); } } else if (groupId.getApplicationId().equals(appId2)) { String entityGroupId = groupId.getTimelineEntityGroupId(); - if (getGroupIds(dagID2, appId2, 100).contains(entityGroupId)) { + if (getGroupIds(dagID2, 100).contains(entityGroupId)) { ++found; } else { Assert.fail("Unexpected group id: " + entityGroupId); @@ -284,7 +285,7 @@ public class TestTimelineCachePluginImpl { Assert.fail("Unexpected appId: " + groupId.getApplicationId()); } } - Assert.assertEquals("All groupIds not returned", 6, found); + Assert.assertEquals("All groupIds not returned", 4, found); } } @@ -374,8 +375,8 @@ public class TestTimelineCachePluginImpl { Assert.assertEquals("All groupIds not returned", 1, found); } - private Set getGroupIds(TezDAGID dagId, ApplicationId appId, int ... allNumDagsPerGroup) { - HashSet groupIds = Sets.newHashSet(dagId.toString(), appId.toString()); + private Set getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) { + HashSet groupIds = Sets.newHashSet(dagId.toString()); for (int numDagsPerGroup : allNumDagsPerGroup) { groupIds.add(dagId.getGroupId(numDagsPerGroup)); }