From common-commits-return-81199-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Apr 10 19:33:02 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9B1DD18067B for ; Tue, 10 Apr 2018 19:33:01 +0200 (CEST) Received: (qmail 96153 invoked by uid 500); 10 Apr 2018 17:32:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 95992 invoked by uid 99); 10 Apr 2018 17:32:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Apr 2018 17:32:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CAD92F678C; Tue, 10 Apr 2018 17:32:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: haibochen@apache.org To: common-commits@hadoop.apache.org Date: Tue, 10 Apr 2018 17:33:03 -0000 Message-Id: <03dc9573bccf4681b07c6e6dac3849f3@git.apache.org> In-Reply-To: <5e8e3feb9fcd45468a872860fb198adb@git.apache.org> References: <5e8e3feb9fcd45468a872860fb198adb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/50] [abbrv] hadoop git commit: YARN-8073 TimelineClientImpl doesn't honor yarn.timeline-service.versions configuration. Contributed by Rohith Sharma K S YARN-8073 TimelineClientImpl doesn't honor yarn.timeline-service.versions configuration. Contributed by Rohith Sharma K S Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/345e7624 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/345e7624 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/345e7624 Branch: refs/heads/YARN-1011 Commit: 345e7624d58a058a1bad666bd1e5ce4b346a9056 Parents: 3087e89 Author: Vrushali C Authored: Wed Apr 4 15:08:03 2018 -0700 Committer: Vrushali C Committed: Wed Apr 4 15:08:03 2018 -0700 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 2 +- .../hadoop/yarn/conf/YarnConfiguration.java | 21 ++++++++++++++++ .../client/api/impl/TimelineClientImpl.java | 23 +++++++++-------- .../yarn/util/timeline/TimelineUtils.java | 3 +-- .../TestCombinedSystemMetricsPublisher.java | 26 ++++++++++++++++++++ .../reader/TimelineReaderServer.java | 2 +- 6 files changed, 61 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index fd93d07..52c13f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -269,7 +269,7 @@ public class JobHistoryEventHandler extends AbstractService LOG.info("Emitting job history data to the timeline service is enabled"); if (YarnConfiguration.timelineServiceEnabled(conf)) { boolean timelineServiceV2Enabled = - ((int) YarnConfiguration.getTimelineServiceVersion(conf) == 2); + YarnConfiguration.timelineServiceV2Enabled(conf); if(timelineServiceV2Enabled) { timelineV2Client = ((MRAppMaster.RunningAppContext)context).getTimelineV2Client(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 42f2cae..41755e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3797,6 +3797,27 @@ public class YarnConfiguration extends Configuration { } /** + * Returns whether the timeline service v.1,5 is enabled via configuration. + * + * @param conf the configuration + * @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a + * version equal to 1.5. + */ + public static boolean timelineServiceV15Enabled(Configuration conf) { + boolean enabled = false; + if (timelineServiceEnabled(conf)) { + Collection versions = getTimelineServiceVersions(conf); + for (Float version : versions) { + if (Float.compare(version, 1.5f) == 0) { + enabled = true; + break; + } + } + } + return enabled; + } + + /** * Returns all the active timeline service versions. It does not check * whether the timeline service itself is enabled. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 44d6d48..88fccd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -82,7 +82,7 @@ public class TimelineClientImpl extends TimelineClient { @VisibleForTesting protected String doAsUser; - private float timelineServiceVersion; + private boolean timelineServiceV15Enabled; private TimelineWriter timelineWriter; private String timelineServiceAddress; @@ -96,15 +96,15 @@ public class TimelineClientImpl extends TimelineClient { } protected void serviceInit(Configuration conf) throws Exception { - timelineServiceVersion = - conf.getFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_VERSION); if (!YarnConfiguration.timelineServiceV1Enabled(conf)) { throw new IOException("Timeline V1 client is not properly configured. " + "Either timeline service is not enabled or version is not set to" + " 1.x"); } - LOG.info("Timeline service address: " + getTimelineServiceAddress()); + + timelineServiceV15Enabled = + YarnConfiguration.timelineServiceV15Enabled(conf); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation realUgi = ugi.getRealUser(); if (realUgi != null) { @@ -126,6 +126,7 @@ public class TimelineClientImpl extends TimelineClient { conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS); } + LOG.info("Timeline service address: " + getTimelineServiceAddress()); super.serviceInit(conf); } @@ -147,7 +148,7 @@ public class TimelineClientImpl extends TimelineClient { protected TimelineWriter createTimelineWriter(Configuration conf, UserGroupInformation ugi, Client webClient, URI uri) throws IOException { - if (Float.compare(this.timelineServiceVersion, 1.5f) == 0) { + if (timelineServiceV15Enabled) { return new FileSystemTimelineWriter( conf, ugi, webClient, uri); } else { @@ -406,10 +407,9 @@ public class TimelineClientImpl extends TimelineClient { public TimelinePutResponse putEntities(ApplicationAttemptId appAttemptId, TimelineEntityGroupId groupId, TimelineEntity... entities) throws IOException, YarnException { - if (Float.compare(this.timelineServiceVersion, 1.5f) != 0) { + if (!timelineServiceV15Enabled) { throw new YarnException( - "This API is not supported under current Timeline Service Version: " - + timelineServiceVersion); + "This API is not supported under current Timeline Service Version:"); } return timelineWriter.putEntities(appAttemptId, groupId, entities); @@ -418,10 +418,9 @@ public class TimelineClientImpl extends TimelineClient { @Override public void putDomain(ApplicationAttemptId appAttemptId, TimelineDomain domain) throws IOException, YarnException { - if (Float.compare(this.timelineServiceVersion, 1.5f) != 0) { + if (!timelineServiceV15Enabled) { throw new YarnException( - "This API is not supported under current Timeline Service Version: " - + timelineServiceVersion); + "This API is not supported under current Timeline Service Version:"); } timelineWriter.putDomain(appAttemptId, domain); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index a0c4b72..800e8ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -126,8 +126,7 @@ public class TimelineUtils { * version equal to 1.5. */ public static boolean timelineServiceV1_5Enabled(Configuration conf) { - return timelineServiceEnabled(conf) && - Math.abs(getTimelineServiceVersion(conf) - 1.5) < 0.00001; + return YarnConfiguration.timelineServiceV15Enabled(conf); } public static TimelineAbout createTimelineAbout(String about) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index 830d01c..5b0c34f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -31,6 +31,7 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.*; @@ -266,6 +267,31 @@ public class TestCombinedSystemMetricsPublisher { runTest(false, false); } + @Test(timeout = 10000) + public void testTimelineServiceConfiguration() + throws Exception { + Configuration config = new Configuration(false); + config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0,1.5"); + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "2.0"); + + Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); + Assert.assertTrue(YarnConfiguration.timelineServiceV15Enabled(config)); + Assert.assertTrue(YarnConfiguration.timelineServiceV1Enabled(config)); + + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0,1"); + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.5"); + Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); + Assert.assertFalse(YarnConfiguration.timelineServiceV15Enabled(config)); + Assert.assertTrue(YarnConfiguration.timelineServiceV1Enabled(config)); + + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0"); + config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.5"); + Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); + Assert.assertFalse(YarnConfiguration.timelineServiceV15Enabled(config)); + Assert.assertFalse(YarnConfiguration.timelineServiceV1Enabled(config)); + } + private void publishEvents(boolean v1Enabled, boolean v2Enabled) { long timestamp = (v1Enabled) ? 1 : 2; int id = (v2Enabled) ? 3 : 4; http://git-wip-us.apache.org/repos/asf/hadoop/blob/345e7624/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java index 3cc24ea..2f85c04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java @@ -235,7 +235,7 @@ public class TimelineReaderServer extends CompositeService { public static void main(String[] args) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, 2.0f); TimelineReaderServer server = startTimelineReaderServer(args, conf); server.join(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org