hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From varunsax...@apache.org
Subject [37/50] [abbrv] hadoop git commit: YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin Lee via Varun Saxena).
Date Sun, 06 Nov 2016 16:31:54 GMT
YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin
Lee via Varun Saxena).

(cherry picked from commit a3ac1c1943518c2a242dd21dac0f17f21bc3cd7f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4de08db4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4de08db4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4de08db4

Branch: refs/heads/YARN-5355-branch-2
Commit: 4de08db4d9ee000e1d311294de95bcf59588b541
Parents: bd32c28
Author: Varun Saxena <varunsaxena@apache.org>
Authored: Tue Jul 12 20:41:52 2016 +0530
Committer: Varun Saxena <varunsaxena@apache.org>
Committed: Sun Nov 6 21:25:25 2016 +0530

----------------------------------------------------------------------
 .../distributedshell/TestDistributedShell.java  | 34 +++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de08db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index a2d82db..300ea67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -83,7 +83,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher;
 import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -92,6 +91,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.rules.Timeout;
 
 public class TestDistributedShell {
@@ -104,6 +104,8 @@ public class TestDistributedShell {
   private FileSystem fs = null;
   private TimelineWriter spyTimelineWriter;
   protected YarnConfiguration conf = null;
+  // location of the filesystem timeline writer for timeline service v.2
+  private String timelineV2StorageDir = null;
   private static final int NUM_NMS = 1;
   private static final float DEFAULT_TIMELINE_VERSION = 1.0f;
   private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
@@ -116,6 +118,8 @@ public class TestDistributedShell {
       = new TimelineVersionWatcher();
   @Rule
   public Timeout globalTimeout = new Timeout(90000);
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
 
   @Before
   public void setup() throws Exception {
@@ -188,6 +192,10 @@ public class TestDistributedShell {
           FileSystemTimelineWriterImpl.class,
           org.apache.hadoop.yarn.server.timelineservice.storage.
               TimelineWriter.class);
+      timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath();
+      // set the file system timeline writer storage directory
+      conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+          timelineV2StorageDir);
     } else {
       Assert.fail("Wrong timeline version number: " + timelineVersion);
     }
@@ -258,7 +266,7 @@ public class TestDistributedShell {
             new Path(conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH)),
             true);
   }
-  
+
   @Test
   public void testDSShellWithDomain() throws Exception {
     testDSShell(true);
@@ -488,22 +496,23 @@ public class TestDistributedShell {
   private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
       boolean defaultFlow, ApplicationReport appReport) throws Exception {
     LOG.info("Started checkTimelineV2 ");
-    // For PoC check in /tmp/timeline_service_data YARN-3264
-    String tmpRoot =
-        FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
-            + "/entities/";
+    // For PoC check using the file-based timeline writer (YARN-3264)
+    String tmpRoot = timelineV2StorageDir + File.separator + "entities" +
+        File.separator;
 
     File tmpRootFolder = new File(tmpRoot);
     try {
       Assert.assertTrue(tmpRootFolder.isDirectory());
       String basePath = tmpRoot +
-          YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+          YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator +
           UserGroupInformation.getCurrentUser().getShortUserName() +
           (defaultFlow ?
-              "/" + appReport.getName() + "/" +
-                  TimelineUtils.DEFAULT_FLOW_VERSION +"/" +
-                  appReport.getStartTime() +"/" :
-              "/test_flow_name/test_flow_version/12345678/") +
+              File.separator + appReport.getName() + File.separator +
+                  TimelineUtils.DEFAULT_FLOW_VERSION + File.separator +
+                  appReport.getStartTime() + File.separator :
+              File.separator + "test_flow_name" + File.separator +
+                  "test_flow_version" + File.separator + "12345678" +
+                  File.separator) +
           appId.toString();
       LOG.info("basePath: " + basePath);
       // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
@@ -615,7 +624,8 @@ public class TestDistributedShell {
 
   private File verifyEntityTypeFileExists(String basePath, String entityType,
       String entityfileName) {
-    String outputDirPathForEntity = basePath + "/" + entityType + "/";
+    String outputDirPathForEntity =
+        basePath + File.separator + entityType + File.separator;
     File outputDirForEntity = new File(outputDirPathForEntity);
     Assert.assertTrue(outputDirForEntity.isDirectory());
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message