Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 935F6200B53 for ; Tue, 12 Jul 2016 17:12:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 91C6C160A56; Tue, 12 Jul 2016 15:12:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B3083160A53 for ; Tue, 12 Jul 2016 17:12:36 +0200 (CEST) Received: (qmail 56688 invoked by uid 500); 12 Jul 2016 15:12:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 56679 invoked by uid 99); 12 Jul 2016 15:12:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jul 2016 15:12:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9B8A3DFF68; Tue, 12 Jul 2016 15:12:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: varunsaxena@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin Lee via Varun Saxena). Date: Tue, 12 Jul 2016 15:12:35 +0000 (UTC) archived-at: Tue, 12 Jul 2016 15:12:37 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 819224dcf -> a3ac1c194 YARN-5354. TestDistributedShell.checkTimelineV2 fails randomly for concurrent tests. (Sangjin Lee via Varun Saxena). Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3ac1c19 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3ac1c19 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3ac1c19 Branch: refs/heads/trunk Commit: a3ac1c1943518c2a242dd21dac0f17f21bc3cd7f Parents: 819224d Author: Varun Saxena Authored: Tue Jul 12 20:41:52 2016 +0530 Committer: Varun Saxena Committed: Tue Jul 12 20:41:52 2016 +0530 ---------------------------------------------------------------------- .../distributedshell/TestDistributedShell.java | 34 +++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3ac1c19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index b9b8c7f..9b99bd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -83,7 +83,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -92,6 +91,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; public class TestDistributedShell { @@ -104,6 +104,8 @@ public class TestDistributedShell { private FileSystem fs = null; private TimelineWriter spyTimelineWriter; protected YarnConfiguration conf = null; + // location of the filesystem timeline writer for timeline service v.2 + private String timelineV2StorageDir = null; private static final int NUM_NMS = 1; private static final float DEFAULT_TIMELINE_VERSION = 1.0f; private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; @@ -116,6 +118,8 @@ public class TestDistributedShell { = new TimelineVersionWatcher(); @Rule public Timeout globalTimeout = new Timeout(90000); + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); @Before public void setup() throws Exception { @@ -188,6 +192,10 @@ public class TestDistributedShell { FileSystemTimelineWriterImpl.class, org.apache.hadoop.yarn.server.timelineservice.storage. TimelineWriter.class); + timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath(); + // set the file system timeline writer storage directory + conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, + timelineV2StorageDir); } else { Assert.fail("Wrong timeline version number: " + timelineVersion); } @@ -259,7 +267,7 @@ public class TestDistributedShell { .get("yarn.timeline-service.leveldb-timeline-store.path")), true); } - + @Test public void testDSShellWithDomain() throws Exception { testDSShell(true); @@ -489,22 +497,23 @@ public class TestDistributedShell { private void checkTimelineV2(boolean haveDomain, ApplicationId appId, boolean defaultFlow, ApplicationReport appReport) throws Exception { LOG.info("Started checkTimelineV2 "); - // For PoC check in /tmp/timeline_service_data YARN-3264 - String tmpRoot = - FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT - + "/entities/"; + // For PoC check using the file-based timeline writer (YARN-3264) + String tmpRoot = timelineV2StorageDir + File.separator + "entities" + + File.separator; File tmpRootFolder = new File(tmpRoot); try { Assert.assertTrue(tmpRootFolder.isDirectory()); String basePath = tmpRoot + - YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" + + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator + UserGroupInformation.getCurrentUser().getShortUserName() + (defaultFlow ? - "/" + appReport.getName() + "/" + - TimelineUtils.DEFAULT_FLOW_VERSION +"/" + - appReport.getStartTime() +"/" : - "/test_flow_name/test_flow_version/12345678/") + + File.separator + appReport.getName() + File.separator + + TimelineUtils.DEFAULT_FLOW_VERSION + File.separator + + appReport.getStartTime() + File.separator : + File.separator + "test_flow_name" + File.separator + + "test_flow_version" + File.separator + "12345678" + + File.separator) + appId.toString(); LOG.info("basePath: " + basePath); // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs @@ -616,7 +625,8 @@ public class TestDistributedShell { private File verifyEntityTypeFileExists(String basePath, String entityType, String entityfileName) { - String outputDirPathForEntity = basePath + "/" + entityType + "/"; + String outputDirPathForEntity = + basePath + File.separator + entityType + File.separator; File outputDirForEntity = new File(outputDirPathForEntity); Assert.assertTrue(outputDirForEntity.isDirectory()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org