tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky. (hitesh)
Date Fri, 07 Nov 2014 01:11:47 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 d69247eed -> 9d2adce53


TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
(hitesh)

(cherry picked from commit 71ca74e0e8e9e59dd1490961cc15482487cdbec8)

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9d2adce5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9d2adce5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9d2adce5

Branch: refs/heads/branch-0.5
Commit: 9d2adce537ea2e55f0863ede69620a8b6c5de180
Parents: d69247e
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Nov 6 16:45:41 2014 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Nov 6 17:09:53 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  5 +++
 .../logging/ats/ATSHistoryLoggingService.java   |  8 ++++
 .../ats/TestATSHistoryLoggingService.java       | 47 +++++++++++++++-----
 .../src/test/resources/log4j.properties         | 19 ++++++++
 4 files changed, 68 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/9d2adce5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ee3f358..730700f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,11 @@
 Apache Tez Change Log
 =====================
 
+Release 0.5.3: Unreleased
+
+ALL CHANGES:
+  TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
+
 Release 0.5.2: Unreleased
 
 INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/tez/blob/9d2adce5/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index cdfbf7c..0108c26 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -119,6 +119,11 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
               // Finish processing events and then return
               interrupted = true;
             }
+
+            if (events.isEmpty()) {
+              continue;
+            }
+
             eventsProcessed += events.size();
             try {
               handleEvents(events);
@@ -242,6 +247,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
           events.get(i).getHistoryEvent());
     }
 
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Sending event batch to Timeline, batchSize=" + events.size());
+    }
     try {
       TimelinePutResponse response =
           timelineClient.putEntities(entities);

http://git-wip-us.apache.org/repos/asf/tez/blob/9d2adce5/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
index 06bc065..a9e00c1 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
@@ -18,6 +18,8 @@
 
 package org.apache.tez.dag.history.logging.ats;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@@ -41,6 +43,8 @@ import static org.mockito.Mockito.when;
 
 public class TestATSHistoryLoggingService {
 
+  private static final Log LOG = LogFactory.getLog(TestATSHistoryLoggingService.class);
+
   private ATSHistoryLoggingService atsHistoryLoggingService;
   private AppContext appContext;
   private Configuration conf;
@@ -56,6 +60,7 @@ public class TestATSHistoryLoggingService {
     conf = new Configuration(false);
     conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
         1000l);
+    conf.setInt(TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH, 2);
     atsInvokeCounter = 0;
     atsEntitiesCounter = 0;
     atsHistoryLoggingService.init(conf);
@@ -67,10 +72,10 @@ public class TestATSHistoryLoggingService {
         new Answer<Object>() {
           @Override
           public Object answer(InvocationOnMock invocation) throws Throwable {
-            atsEntitiesCounter += invocation.getArguments().length;
             ++atsInvokeCounter;
+            atsEntitiesCounter += invocation.getArguments().length;
             try {
-              Thread.sleep(500);
+              Thread.sleep(500l);
             } catch (InterruptedException e) {
               // do nothing
             }
@@ -98,21 +103,41 @@ public class TestATSHistoryLoggingService {
     }
 
     try {
-      Thread.sleep(500l);
+      Thread.sleep(2500l);
     } catch (InterruptedException e) {
       // Do nothing
     }
     atsHistoryLoggingService.stop();
 
-    Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter >=
10);
-    Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter <
50);
-    Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter >= 1);
-    Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter <= 10);
+    LOG.info("ATS entitiesSent=" + atsEntitiesCounter
+        + ", timelineInvocations=" + atsInvokeCounter);
+
+    Assert.assertTrue(atsEntitiesCounter >= 4);
+    Assert.assertTrue(atsEntitiesCounter < 20);
+
+  }
+
+  @Test(timeout=20000)
+  public void testATSEventBatching() {
+    TezDAGID tezDAGID = TezDAGID.getInstance(
+        ApplicationId.newInstance(100l, 1), 1);
+    DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID,
+        new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
+
+    for (int i = 0; i < 100; ++i) {
+      atsHistoryLoggingService.handle(historyEvent);
+    }
+
+    try {
+      Thread.sleep(1000l);
+    } catch (InterruptedException e) {
+      // Do nothing
+    }
+    LOG.info("ATS entitiesSent=" + atsEntitiesCounter
+        + ", timelineInvocations=" + atsInvokeCounter);
 
-    Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter
-            + ", ATSEntitiesCounter = " + atsEntitiesCounter,
-        atsEntitiesCounter >=
-            ((atsInvokeCounter-1) * TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT));
+    Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter);
+    Assert.assertEquals(atsEntitiesCounter/2, atsInvokeCounter);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/9d2adce5/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) -
%m%n


Mime
View raw message