Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E3DA7200BE3 for ; Thu, 22 Dec 2016 20:31:42 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id E258A160B35; Thu, 22 Dec 2016 19:31:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B95DE160B27 for ; Thu, 22 Dec 2016 20:31:41 +0100 (CET) Received: (qmail 4563 invoked by uid 500); 22 Dec 2016 19:31:30 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 909 invoked by uid 99); 22 Dec 2016 19:31:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Dec 2016 19:31:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 02B9ADFCF4; Thu, 22 Dec 2016 19:31:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Thu, 22 Dec 2016 19:32:05 -0000 Message-Id: In-Reply-To: <00a1781260504645b47d449124968b62@git.apache.org> References: <00a1781260504645b47d449124968b62@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [39/51] [abbrv] hadoop git commit: YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He archived-at: Thu, 22 Dec 2016 19:31:43 -0000 YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b39605fb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b39605fb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b39605fb Branch: refs/heads/yarn-native-services Commit: b39605fbd154e00db0935b1f92e2c1420101ddc5 Parents: cef1ce4 Author: Gour Saha Authored: Fri Dec 16 12:03:51 2016 -0800 Committer: Jian He Committed: Thu Dec 22 11:09:38 2016 -0800 ---------------------------------------------------------------------- .../api/impl/ApplicationApiService.java | 82 ++++++++++++-------- .../org/apache/slider/client/SliderClient.java | 62 +++++++++++---- .../slider/common/params/ActionStatusArgs.java | 4 + 3 files changed, 103 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index c4f5d43..b11da2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -52,6 +52,8 @@ import org.apache.commons.lang.SerializationUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.services.api.ApplicationApi; @@ -771,7 +773,7 @@ public class ApplicationApiService implements ApplicationApi { // Get all applications in a specific state - lighter projection. For full // detail, call getApplication on a specific app. - Set applications; + Set applications; try { if (StringUtils.isNotEmpty(state)) { ApplicationStatus appStatus = new ApplicationStatus(); @@ -793,13 +795,12 @@ public class ApplicationApiService implements ApplicationApi { Set apps = new HashSet(); if (applications.size() > 0) { try { - for (String app : applications) { + for (ApplicationReport app : applications) { Application application = new Application(); - // TODO: Need to get lifetime, launch-time and privileged container - // status from YARN - application.setLifetime(null); - application.setLaunchTime(new Date()); - application.setName(app); + application.setLifetime(app.getApplicationTimeouts().get( + ApplicationTimeoutType.LIFETIME).getRemainingTime()); + application.setLaunchTime(new Date(app.getStartTime())); + application.setName(app.getName()); // Containers not required, setting to null to avoid empty list application.setContainers(null); apps.add(application); @@ -930,9 +931,7 @@ public class ApplicationApiService implements ApplicationApi { app.setLaunchTime(appStatus.get("createTime") == null ? null : new Date(appStatus.get("createTime").getAsLong())); - // lifetime - set it to unlimited for now - // TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN - app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); + app.setLifetime(queryLifetime(appName)); // Quicklinks Map appQuicklinks = new HashMap<>(); @@ -1062,6 +1061,24 @@ public class ApplicationApiService implements ApplicationApi { return object.get(key) == null ? null : object.get(key).getAsJsonObject(); } + private long queryLifetime(String appName) { + try { + return invokeSliderClientRunnable( + new SliderClientContextRunnable() { + @Override + public Long run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + ApplicationReport report = sliderClient.findInstance(appName); + return report.getApplicationTimeouts() + .get(ApplicationTimeoutType.LIFETIME).getRemainingTime(); + } + }); + } catch (Exception e) { + logger.error("Error when querying lifetime for " + appName, e); + return DEFAULT_UNLIMITED_LIFETIME; + } + } + private JsonObject getSliderApplicationStatus(final String appName) throws IOException, YarnException, InterruptedException { @@ -1142,36 +1159,37 @@ public class ApplicationApiService implements ApplicationApi { }); } - private Set getSliderApplications(final String state) + private Set getSliderApplications(final String state) throws IOException, YarnException, InterruptedException { return getSliderApplications(false, state); } - private Set getSliderApplications(final boolean liveOnly) + private Set getSliderApplications(final boolean liveOnly) throws IOException, YarnException, InterruptedException { return getSliderApplications(liveOnly, null); } - private Set getSliderApplications(final boolean liveOnly, - final String state) throws IOException, YarnException, - InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable>() { - @Override - public Set run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - Set apps; - ActionListArgs listArgs = new ActionListArgs(); - if (liveOnly) { - apps = sliderClient.getApplicationList(null); - } else if (StringUtils.isNotEmpty(state)) { - listArgs.state = state; - apps = sliderClient.getApplicationList(null, listArgs); - } else { - apps = sliderClient.getApplicationList(null, listArgs); - } - return apps; - } - }); + private Set getSliderApplications(final boolean liveOnly, + final String state) + throws IOException, YarnException, InterruptedException { + return invokeSliderClientRunnable( + new SliderClientContextRunnable>() { + @Override + public Set run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + Set apps; + ActionListArgs listArgs = new ActionListArgs(); + if (liveOnly) { + apps = sliderClient.getApplicationList(null); + } else if (StringUtils.isNotEmpty(state)) { + listArgs.state = state; + apps = sliderClient.getApplicationList(null, listArgs); + } else { + apps = sliderClient.getApplicationList(null, listArgs); + } + return apps; + } + }); } @DELETE http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef45d10..1c126ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -178,6 +179,7 @@ import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.io.Console; import java.io.File; import java.io.FileNotFoundException; @@ -185,6 +187,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; +import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; @@ -2706,11 +2709,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException { - Set appInstances = getApplicationList(clustername, args); - // getApplicationList never returns null - return !appInstances.isEmpty() ? EXIT_SUCCESS - : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS - : EXIT_FALSE); + Set appInstances = getApplicationList(clustername, args); + if (!appInstances.isEmpty()) { + return EXIT_SUCCESS; + } else { + return EXIT_FALSE; + } } /** @@ -2723,8 +2727,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws IOException * @throws YarnException */ - public Set getApplicationList(String clustername) throws IOException, - YarnException { + public Set getApplicationList(String clustername) + throws IOException, YarnException { ActionListArgs args = new ActionListArgs(); args.live = true; return getApplicationList(clustername, args); @@ -2743,8 +2747,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws UnknownApplicationInstanceException * if a specific instance was named but it was not found */ - public Set getApplicationList(String clustername, ActionListArgs args) - throws IOException, YarnException { + public Set getApplicationList(String clustername, + ActionListArgs args) throws IOException, YarnException { if (args.help) { actionHelp(ACTION_LIST); // the above call throws an exception so the return is not really required @@ -2830,13 +2834,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } // at this point there is either the entire list or a stripped down instance - Set listedInstances = new HashSet(); + Set listedInstances = new HashSet(); for (String name : persistentInstances.keySet()) { ApplicationReport report = reportMap.get(name); if (!listOnlyInState || report != null) { // list the details if all were requested, or the filtering contained // a report - listedInstances.add(name); + listedInstances.add(report); // containers will be non-null when only one instance is requested String details = instanceDetailsToString(name, report, containers, version, components, verbose); @@ -3055,7 +3059,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws YarnException YARN issues * @throws IOException IO problems */ - private ApplicationReport findInstance(String appname) + public ApplicationReport findInstance(String appname) throws YarnException, IOException { return yarnAppListClient.findInstance(appname); } @@ -3106,6 +3110,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @VisibleForTesting public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws YarnException, IOException { + if (statusArgs.lifetime) { + queryAndPrintLifetime(clustername); + return EXIT_SUCCESS; + } + ClusterDescription status = verifyAndGetClusterDescription(clustername); String outfile = statusArgs.getOutput(); if (outfile == null) { @@ -3122,6 +3131,32 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return verifyAndGetClusterDescription(clustername).toJsonString(); } + private void queryAndPrintLifetime(String appName) + throws YarnException, IOException { + ApplicationReport appReport = findInstance(appName); + if (appReport == null) { + throw new YarnException("No application found for " + appName); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter timeoutStr = + new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + try { + ApplicationTimeout lifetime = appReport.getApplicationTimeouts() + .get(ApplicationTimeoutType.LIFETIME); + if (lifetime.getRemainingTime() == -1L) { + timeoutStr.append(appName + " has no lifetime configured."); + } else { + timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME); + timeoutStr.print(" expires at : " + lifetime.getExpiryTime()); + timeoutStr.println( + ".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds"); + } + System.out.println(baos.toString("UTF-8")); + } finally { + timeoutStr.close(); + } + } + private ClusterDescription verifyAndGetClusterDescription(String clustername) throws YarnException, IOException { verifyBindingsDefined(); @@ -3547,7 +3582,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws IOException */ @VisibleForTesting - public List getApplications() throws YarnException, IOException { + public List getApplications() + throws YarnException, IOException { return yarnClient.getApplications(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java index 00178df..6fbd96d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java @@ -35,6 +35,10 @@ public class ActionStatusArgs extends AbstractActionArgs { description = "Output file for the status information") public String output; + @Parameter(names = {ARG_LIFETIME}, + description = "Lifetime of the application from the time of request") + public boolean lifetime; + public String getOutput() { return output; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org