hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sj...@apache.org
Subject [46/50] [abbrv] hadoop git commit: YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin Lee via Varun Saxena).
Date Sat, 16 Jul 2016 03:37:07 GMT
YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin
Lee via Varun Saxena).

(cherry picked from commit a3ac1c1943518c2a242dd21dac0f17f21bc3cd7f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77f05ecf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77f05ecf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77f05ecf

Branch: refs/heads/YARN-5355-branch-2
Commit: 77f05ecf89e8e78d266d8becb12a93e3064a1bc2
Parents: bb0a24c
Author: Varun Saxena <varunsaxena@apache.org>
Authored: Tue Jul 12 20:41:52 2016 +0530
Committer: Sangjin Lee <sjlee@apache.org>
Committed: Fri Jul 15 20:28:04 2016 -0700

----------------------------------------------------------------------
 .../distributedshell/TestDistributedShell.java  | 34 +++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f05ecf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index b9b8c7f..9b99bd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -83,7 +83,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher;
 import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -92,6 +91,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.rules.Timeout;
 
 public class TestDistributedShell {
@@ -104,6 +104,8 @@ public class TestDistributedShell {
   private FileSystem fs = null;
   private TimelineWriter spyTimelineWriter;
   protected YarnConfiguration conf = null;
+  // location of the filesystem timeline writer for timeline service v.2
+  private String timelineV2StorageDir = null;
   private static final int NUM_NMS = 1;
   private static final float DEFAULT_TIMELINE_VERSION = 1.0f;
   private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
@@ -116,6 +118,8 @@ public class TestDistributedShell {
       = new TimelineVersionWatcher();
   @Rule
   public Timeout globalTimeout = new Timeout(90000);
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
 
   @Before
   public void setup() throws Exception {
@@ -188,6 +192,10 @@ public class TestDistributedShell {
           FileSystemTimelineWriterImpl.class,
           org.apache.hadoop.yarn.server.timelineservice.storage.
               TimelineWriter.class);
+      timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath();
+      // set the file system timeline writer storage directory
+      conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+          timelineV2StorageDir);
     } else {
       Assert.fail("Wrong timeline version number: " + timelineVersion);
     }
@@ -259,7 +267,7 @@ public class TestDistributedShell {
                 .get("yarn.timeline-service.leveldb-timeline-store.path")),
             true);
   }
-  
+
   @Test
   public void testDSShellWithDomain() throws Exception {
     testDSShell(true);
@@ -489,22 +497,23 @@ public class TestDistributedShell {
   private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
       boolean defaultFlow, ApplicationReport appReport) throws Exception {
     LOG.info("Started checkTimelineV2 ");
-    // For PoC check in /tmp/timeline_service_data YARN-3264
-    String tmpRoot =
-        FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
-            + "/entities/";
+    // For PoC check using the file-based timeline writer (YARN-3264)
+    String tmpRoot = timelineV2StorageDir + File.separator + "entities" +
+        File.separator;
 
     File tmpRootFolder = new File(tmpRoot);
     try {
       Assert.assertTrue(tmpRootFolder.isDirectory());
       String basePath = tmpRoot +
-          YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+          YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator +
           UserGroupInformation.getCurrentUser().getShortUserName() +
           (defaultFlow ?
-              "/" + appReport.getName() + "/" +
-                  TimelineUtils.DEFAULT_FLOW_VERSION +"/" +
-                  appReport.getStartTime() +"/" :
-              "/test_flow_name/test_flow_version/12345678/") +
+              File.separator + appReport.getName() + File.separator +
+                  TimelineUtils.DEFAULT_FLOW_VERSION + File.separator +
+                  appReport.getStartTime() + File.separator :
+              File.separator + "test_flow_name" + File.separator +
+                  "test_flow_version" + File.separator + "12345678" +
+                  File.separator) +
           appId.toString();
       LOG.info("basePath: " + basePath);
       // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
@@ -616,7 +625,8 @@ public class TestDistributedShell {
 
   private File verifyEntityTypeFileExists(String basePath, String entityType,
       String entityfileName) {
-    String outputDirPathForEntity = basePath + "/" + entityType + "/";
+    String outputDirPathForEntity =
+        basePath + File.separator + entityType + File.separator;
     File outputDirForEntity = new File(outputDirPathForEntity);
     Assert.assertTrue(outputDirForEntity.isDirectory());
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message