tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject tez git commit: TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. (Jonathan Eagles via jlowe) (cherry picked from commit 1468457e671dfcf2780cf17d2655e086aef120c9)
Date Wed, 17 Aug 2016 15:15:44 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.8 119c47621 -> 90519d9f7


TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
(Jonathan Eagles via jlowe)
(cherry picked from commit 1468457e671dfcf2780cf17d2655e086aef120c9)

Conflicts:

	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/90519d9f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/90519d9f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/90519d9f

Branch: refs/heads/branch-0.8
Commit: 90519d9f7b8e3358279daaec0e05fd878d131d23
Parents: 119c476
Author: Jason Lowe <jlowe@apache.org>
Authored: Wed Aug 17 15:15:31 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Wed Aug 17 15:15:31 2016 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../apache/tez/dag/history/utils/DAGUtils.java  |  3 +-
 .../ats/TestHistoryEventTimelineConversion.java | 34 ++++++++++++++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b78f48d..87adaa2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
   TEZ-3352. MRInputHelpers getStringProperty() should not fail if property value is null.
   TEZ-3409. Log dagId along with other information when submitting a dag.
   TEZ-3384. Fix TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test.
@@ -499,6 +500,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
   TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container).
   TEZ-3223. Support a NullHistoryLogger to disable history logging if needed.
   TEZ-3293. Fetch failures can cause a shuffle hang waiting for memory merge that never starts.

http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index f1ac0ab..d8d2407 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -475,7 +475,8 @@ public class DAGUtils {
   }
 
   public static Map<String, String> convertConfigurationToATSMap(Configuration conf)
{
-    Iterator<Entry<String, String>> iter = conf.iterator();
+    // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500
+    Iterator<Entry<String, String>> iter = new Configuration(conf).iterator();
     Map<String, String> atsConf = new TreeMap<String, String>();
     while (iter.hasNext()) {
       Entry<String, String> entry = iter.next();

http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index dabd6a5..62fb335 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -232,6 +233,39 @@ public class TestHistoryEventTimelineConversion {
 
   }
 
+  @Test(timeout = 5000)
+  public void testConvertAppLaunchedEventConcurrentModificationException()
+      throws InterruptedException {
+    long launchTime = random.nextLong();
+    long submitTime = random.nextLong();
+    final Configuration conf = new Configuration(false);
+    conf.set("foo", "bar");
+    conf.set("applicationId", "1234");
+    final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+    Thread confChanger = new Thread() {
+      @Override
+      public void run() {
+        int i = 1;
+        while (!shutdown.get()) {
+          // trigger an actual change to test concurrency with Configuration#iterator
+          conf.set("test" + i++, "test");
+        }
+      }
+    };
+
+    confChanger.start();
+    try {
+      MockVersionInfo mockVersionInfo = new MockVersionInfo();
+      AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
+          submitTime, user, conf, mockVersionInfo);
+      HistoryEventTimelineConversion.convertToTimelineEntity(event);
+    } finally {
+      shutdown.set(true);
+      confChanger.join();
+    }
+  }
+
   @SuppressWarnings("unchecked")
   @Test(timeout = 5000)
   public void testConvertAppLaunchedEvent() {


Mime
View raw message