Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 105EF1068F for ; Wed, 3 Dec 2014 05:23:46 +0000 (UTC) Received: (qmail 28072 invoked by uid 500); 3 Dec 2014 05:23:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 27632 invoked by uid 500); 3 Dec 2014 05:23:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 26552 invoked by uid 99); 3 Dec 2014 05:23:34 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Dec 2014 05:23:34 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id D3639A07791; Wed, 3 Dec 2014 05:23:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Date: Wed, 03 Dec 2014 05:23:48 -0000 Message-Id: <86ebe483867d49249e7e41c7b767654c@git.apache.org> In-Reply-To: <1d780480d3a947b5bd82f2b6d1bd67f3@git.apache.org> References: <1d780480d3a947b5bd82f2b6d1bd67f3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/50] [abbrv] hadoop git commit: YARN-2165. Added the sanity check for the numeric configuration values of the timeline service. Contributed by Vasanth kumar RJ. YARN-2165. Added the sanity check for the numeric configuration values of the timeline service. Contributed by Vasanth kumar RJ. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f1454cc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f1454cc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f1454cc Branch: refs/heads/HDFS-EC Commit: 8f1454cc6d46afa057770a39aecc151c5f048b57 Parents: 978736d Author: Zhijie Shen Authored: Wed Nov 26 12:09:34 2014 -0800 Committer: Zhijie Shen Committed: Wed Nov 26 12:09:34 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../client/api/impl/TimelineClientImpl.java | 13 ++++ .../src/main/resources/yarn-default.xml | 3 +- .../client/api/impl/TestTimelineClient.java | 23 +++++++ .../ApplicationHistoryClientService.java | 8 +++ .../server/timeline/LeveldbTimelineStore.java | 27 ++++++++ .../TestApplicationHistoryServer.java | 14 ++++ .../timeline/TestLeveldbTimelineStore.java | 72 +++++++++++++++++++- 8 files changed, 161 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e9fa63e..e8b82a0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -98,6 +98,9 @@ Release 2.7.0 - UNRELEASED YARN-2404. Removed ApplicationAttemptState and ApplicationState class in RMStateStore. (Tsuyoshi OZAWA via jianhe) + YARN-2165. Added the sanity check for the numeric configuration values of + the timeline service. (Vasanth kumar RJ via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 78901c3..605d60b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -67,6 +67,7 @@ import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientRequest; @@ -141,6 +142,18 @@ public class TimelineClientImpl extends TimelineClient { // Constructor with default retry settings public TimelineClientConnectionRetry(Configuration conf) { + Preconditions.checkArgument(conf.getInt( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) >= -1, + "%s property value should be greater than or equal to -1", + YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES); + Preconditions + .checkArgument( + conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS) > 0, + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); maxRetries = conf.getInt( YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0ff989e..2f549da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1355,7 +1355,8 @@ - Default maximum number of retires for timeline servive client. + Default maximum number of retires for timeline servive client + and value -1 means no limit. yarn.timeline-service.client.max-retries 30 http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 3b4d6bd..7da3bbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -154,6 +154,29 @@ public class TestTimelineClient { @Test public void testCheckRetryCount() throws Exception { + try { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + -2); + createTimelineClient(conf); + Assert.fail(); + } catch(IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES)); + } + + try { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + 0); + createTimelineClient(conf); + Assert.fail(); + } catch(IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS)); + } int newMaxRetries = 5; long newIntervalMs = 500; YarnConfiguration conf = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 2334fde..4c04c5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider; +import com.google.common.base.Preconditions; + public class ApplicationHistoryClientService extends AbstractService { private static final Log LOG = LogFactory .getLog(ApplicationHistoryClientService.class); @@ -84,6 +86,12 @@ public class ApplicationHistoryClientService extends AbstractService { YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + Preconditions.checkArgument(conf.getInt( + YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0, + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT); + server = rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler, address, conf, null, conf.getInt( http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java index 33deb80..ba75c14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java @@ -76,6 +76,7 @@ import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** *

An implementation of an application timeline store backed by leveldb.

@@ -194,6 +195,32 @@ public class LeveldbTimelineStore extends AbstractService @Override @SuppressWarnings("unchecked") protected void serviceInit(Configuration conf) throws Exception { + Preconditions.checkArgument(conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_TTL_MS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_TTL_MS) > 0, + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_TTL_MS); + Preconditions.checkArgument(conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS) > 0, + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS); + Preconditions.checkArgument(conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE) >= 0, + "%s property value should be greater than or equal to zero", + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE); + Preconditions.checkArgument(conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE) > 0, + " %s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE); + Preconditions.checkArgument(conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE) > 0, + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE); + Options options = new Options(); options.createIfMissing(true); options.cacheSize(conf.getLong( http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 7a4062d..9301221 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.junit.After; +import org.junit.Assert; import org.junit.Test; import java.util.HashMap; @@ -49,6 +50,19 @@ public class TestApplicationHistoryServer { MemoryTimelineStore.class, TimelineStore.class); config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0"); try { + try { + historyServer.init(config); + config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, + 0); + historyServer.start(); + fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT)); + } + config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT); + historyServer = new ApplicationHistoryServer(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); assertEquals(5, historyServer.getServices().size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f1454cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java index d266aa2..15edecd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java @@ -311,7 +311,77 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { e.getMessage().contains("Incompatible version for timeline store")); } } - + + @Test + public void testValidateConfig() throws IOException { + Configuration copyConfig = new YarnConfiguration(config); + try { + Configuration newConfig = new YarnConfiguration(copyConfig); + newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0); + config = newConfig; + restartTimelineStore(); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); + } + try { + Configuration newConfig = new YarnConfiguration(copyConfig); + newConfig.setLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0); + config = newConfig; + restartTimelineStore(); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); + } + try { + Configuration newConfig = new YarnConfiguration(copyConfig); + newConfig.setLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1); + config = newConfig; + restartTimelineStore(); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); + } + try { + Configuration newConfig = new YarnConfiguration(copyConfig); + newConfig + .setLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE, + 0); + config = newConfig; + restartTimelineStore(); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert + .assertTrue(e + .getMessage().contains( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); + } + try { + Configuration newConfig = new YarnConfiguration(copyConfig); + newConfig + .setLong( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, + 0); + config = newConfig; + restartTimelineStore(); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert + .assertTrue(e + .getMessage() + .contains( + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); + } + config = copyConfig; + restartTimelineStore(); + } + private void restartTimelineStore() throws IOException { // need to close so leveldb releases database lock if (store != null) {