Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1919E10937 for ; Fri, 10 Apr 2015 04:25:10 +0000 (UTC) Received: (qmail 83426 invoked by uid 500); 10 Apr 2015 04:25:01 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 82998 invoked by uid 500); 10 Apr 2015 04:25:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 79940 invoked by uid 99); 10 Apr 2015 04:24:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Apr 2015 04:24:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3400BE00C4; Fri, 10 Apr 2015 04:24:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Fri, 10 Apr 2015 04:25:20 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [24/47] hadoop git commit: YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai. YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai. Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8118c95f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8118c95f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8118c95f Branch: refs/heads/YARN-2928 Commit: 8118c95f8cc253a2da0e449fc408705272ab5ea5 Parents: 1c7c608 Author: Hitesh Shah Authored: Wed Apr 8 14:13:10 2015 -0700 Committer: Zhijie Shen Committed: Thu Apr 9 21:14:38 2015 -0700 ---------------------------------------------------------------------- .../jobhistory/TestJobHistoryEventHandler.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 52 ++++++++- hadoop-yarn-project/CHANGES.txt | 3 + .../distributedshell/TestDistributedShell.java | 2 +- .../hadoop/yarn/server/MiniYARNCluster.java | 6 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++ 6 files changed, 172 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index 43e3dbe..de260c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler { long currentTime = System.currentTimeMillis(); try { yarnCluster = new MiniYARNCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); jheh.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index c2ef128..eab9026 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -35,6 +35,52 @@ import org.junit.Test; public class TestMRTimelineEventHandling { @Test + public void testTimelineServiceStartInMiniCluster() throws Exception { + Configuration conf = new YarnConfiguration(); + + /* + * Timeline service should not start if the config is set to false + * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true); + MiniMRYarnCluster cluster = null; + try { + cluster = new MiniMRYarnCluster( + TestJobHistoryEventHandler.class.getSimpleName(), 1); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); + cluster = null; + try { + cluster = new MiniMRYarnCluster( + TestJobHistoryEventHandler.class.getSimpleName(), 1); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + } + + @Test public void testMRTimelineEventHandling() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); @@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling { cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index aaab195..f47be48 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -218,6 +218,9 @@ Release 2.8.0 - UNRELEASED YARN-3459. Fix failiure of TestLog4jWarningErrorMetricsAppender. (Varun Vasudev via wangda) + YARN-2890. MiniYarnCluster should turn on timeline service if + configured to do so. (Mit Desai via hitesh) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index cc5f5e2..b887c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -123,7 +123,7 @@ public class TestDistributedShell { if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1, - numNodeManager, 1, 1, enableATSV1); + numNodeManager, 1, 1); yarnCluster.init(conf); yarnCluster.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 365e0bb..f8b27b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -262,8 +261,9 @@ public class MiniYARNCluster extends CompositeService { addService(new NodeManagerWrapper(index)); } - if (enableAHS) { - addService(new ApplicationHistoryServerWrapper()); + if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { + addService(new ApplicationHistoryServerWrapper()); } super.serviceInit( http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118c95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java new file mode 100644 index 0000000..8a3c9e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Test; + +public class TestMiniYarnCluster { + @Test + public void testTimelineServiceStartInMiniCluster() throws Exception { + Configuration conf = new YarnConfiguration(); + int numNodeManagers = 1; + int numLocalDirs = 1; + int numLogDirs = 1; + boolean enableAHS; + + /* + * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS flag == false + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = false; + MiniYARNCluster cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + + /* + * Timeline service should start if TIMELINE_SERVICE_ENABLED == true + * and enableAHS == false + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + enableAHS = false; + cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //Timeline service may sometime take a while to get started + int wait = 0; + while(cluster.getApplicationHistoryServer() == null && wait < 20) { + Thread.sleep(500); + wait++; + } + //verify that the timeline service is started. + Assert.assertNotNull("Timeline Service should have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + /* + * Timeline service should start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS == true + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = true; + cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //Timeline service may sometime take a while to get started + int wait = 0; + while(cluster.getApplicationHistoryServer() == null && wait < 20) { + Thread.sleep(500); + wait++; + } + //verify that the timeline service is started. + Assert.assertNotNull("Timeline Service should have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + } +}