Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 618DA1081C for ; Mon, 17 Nov 2014 19:22:12 +0000 (UTC) Received: (qmail 84338 invoked by uid 500); 17 Nov 2014 19:22:12 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 84227 invoked by uid 500); 17 Nov 2014 19:22:12 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 83419 invoked by uid 99); 17 Nov 2014 19:22:11 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Nov 2014 19:22:11 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id D046698DE58; Mon, 17 Nov 2014 19:22:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jeagles@apache.org To: commits@tez.apache.org Date: Mon, 17 Nov 2014 19:22:34 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [25/50] [abbrv] tez git commit: TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky. (hitesh) TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/71ca74e0 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/71ca74e0 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/71ca74e0 Branch: refs/heads/TEZ-8 Commit: 71ca74e0e8e9e59dd1490961cc15482487cdbec8 Parents: 7b55e38 Author: Hitesh Shah Authored: Thu Nov 6 16:45:41 2014 -0800 Committer: Hitesh Shah Committed: Thu Nov 6 16:45:41 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 5 +++ .../logging/ats/ATSHistoryLoggingService.java | 8 ++++ .../ats/TestATSHistoryLoggingService.java | 47 +++++++++++++++----- .../src/test/resources/log4j.properties | 19 ++++++++ 4 files changed, 68 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index adc1a85..2e58f0a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,11 @@ ALL CHANGES: TEZ-1733. TezMerger should sort FileChunks on size when merging TEZ-1738. Tez tfile parser for log parsing +Release 0.5.3: Unreleased + +ALL CHANGES: + TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky. + Release 0.5.2: Unreleased INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java index cdfbf7c..0108c26 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java @@ -119,6 +119,11 @@ public class ATSHistoryLoggingService extends HistoryLoggingService { // Finish processing events and then return interrupted = true; } + + if (events.isEmpty()) { + continue; + } + eventsProcessed += events.size(); try { handleEvents(events); @@ -242,6 +247,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService { events.get(i).getHistoryEvent()); } + if (LOG.isDebugEnabled()) { + LOG.debug("Sending event batch to Timeline, batchSize=" + events.size()); + } try { TimelinePutResponse response = timelineClient.putEntities(entities); http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java index 06bc065..a9e00c1 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java @@ -18,6 +18,8 @@ package org.apache.tez.dag.history.logging.ats; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -41,6 +43,8 @@ import static org.mockito.Mockito.when; public class TestATSHistoryLoggingService { + private static final Log LOG = LogFactory.getLog(TestATSHistoryLoggingService.class); + private ATSHistoryLoggingService atsHistoryLoggingService; private AppContext appContext; private Configuration conf; @@ -56,6 +60,7 @@ public class TestATSHistoryLoggingService { conf = new Configuration(false); conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS, 1000l); + conf.setInt(TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH, 2); atsInvokeCounter = 0; atsEntitiesCounter = 0; atsHistoryLoggingService.init(conf); @@ -67,10 +72,10 @@ public class TestATSHistoryLoggingService { new Answer() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { - atsEntitiesCounter += invocation.getArguments().length; ++atsInvokeCounter; + atsEntitiesCounter += invocation.getArguments().length; try { - Thread.sleep(500); + Thread.sleep(500l); } catch (InterruptedException e) { // do nothing } @@ -98,21 +103,41 @@ public class TestATSHistoryLoggingService { } try { - Thread.sleep(500l); + Thread.sleep(2500l); } catch (InterruptedException e) { // Do nothing } atsHistoryLoggingService.stop(); - Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter >= 10); - Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter < 50); - Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter >= 1); - Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter <= 10); + LOG.info("ATS entitiesSent=" + atsEntitiesCounter + + ", timelineInvocations=" + atsInvokeCounter); + + Assert.assertTrue(atsEntitiesCounter >= 4); + Assert.assertTrue(atsEntitiesCounter < 20); + + } + + @Test(timeout=20000) + public void testATSEventBatching() { + TezDAGID tezDAGID = TezDAGID.getInstance( + ApplicationId.newInstance(100l, 1), 1); + DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, + new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1")); + + for (int i = 0; i < 100; ++i) { + atsHistoryLoggingService.handle(historyEvent); + } + + try { + Thread.sleep(1000l); + } catch (InterruptedException e) { + // Do nothing + } + LOG.info("ATS entitiesSent=" + atsEntitiesCounter + + ", timelineInvocations=" + atsInvokeCounter); - Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter - + ", ATSEntitiesCounter = " + atsEntitiesCounter, - atsEntitiesCounter >= - ((atsInvokeCounter-1) * TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT)); + Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter); + Assert.assertEquals(atsEntitiesCounter/2, atsInvokeCounter); } } http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties new file mode 100644 index 0000000..531b68b --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n