hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [3/4] hadoop git commit: YARN-8372. Distributed shell app master should not release containers when shutdown if keep-container is true. (Suma Shivaprasad via wangda)
Date Fri, 01 Jun 2018 23:02:32 GMT
YARN-8372. Distributed shell app master should not release containers when shutdown if keep-container
is true. (Suma Shivaprasad via wangda)

Change-Id: Ief04d1ca865621f348fba4ac85fa78bc47465904
(cherry picked from commit 8956e5b8db3059e0872e49f59adc6affc76e2274)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed205764
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed205764
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed205764

Branch: refs/heads/branch-3.1
Commit: ed20576413c5ba6f2437d316d2904cfa9dda1958
Parents: 1cee85d
Author: Wangda Tan <wangda@apache.org>
Authored: Fri Jun 1 14:05:02 2018 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Fri Jun 1 15:47:59 2018 -0700

----------------------------------------------------------------------
 .../distributedshell/ApplicationMaster.java     | 24 +++++++++++++++++---
 .../applications/distributedshell/Client.java   | 14 ++++++++----
 2 files changed, 31 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed205764/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index bc018b1..76fa38f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -269,6 +269,8 @@ public class ApplicationMaster {
   private String containerResourceProfile = "";
   Map<String, Resource> resourceProfiles;
 
+  private boolean keepContainersAcrossAttempts = false;
+
   // Counter for completed containers ( complete denotes successful or failed )
   private AtomicInteger numCompletedContainers = new AtomicInteger();
   // Allocated container count so that we know how many containers has the RM
@@ -483,6 +485,13 @@ public class ApplicationMaster {
             + " the number of container retry attempts");
     opts.addOption("placement_spec", true, "Placement specification");
     opts.addOption("debug", false, "Dump out debug information");
+    opts.addOption("keep_containers_across_application_attempts", false,
+        "Flag to indicate whether to keep containers across application "
+            + "attempts."
+            + " If the flag is true, running containers will not be killed when"
+            + " application attempt fails and these containers will be "
+            + "retrieved by"
+            + " the new application attempt ");
 
     opts.addOption("help", false, "Print usage");
     CommandLine cliParser = new GnuParser().parse(opts, args);
@@ -646,6 +655,9 @@ public class ApplicationMaster {
     containerResourceProfile =
         cliParser.getOptionValue("container_resource_profile", "");
 
+    keepContainersAcrossAttempts = cliParser.hasOption(
+        "keep_containers_across_application_attempts");
+
     if (this.placementSpecs == null) {
       numTotalContainers = Integer.parseInt(cliParser.getOptionValue(
           "num_containers", "1"));
@@ -1152,9 +1164,15 @@ public class ApplicationMaster {
       }
     }
 
-    @Override
-    public void onShutdownRequest() {
-      done = true;
+    @Override public void onShutdownRequest() {
+      if (keepContainersAcrossAttempts) {
+        LOG.info("Shutdown request received. Ignoring since "
+            + "keep_containers_across_application_attempts is enabled");
+      } else{
+        LOG.info("Shutdown request received. Processing since "
+            + "keep_containers_across_application_attempts is disabled");
+        done = true;
+      }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed205764/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 61879d0..976e6a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -332,10 +332,12 @@ public class Client {
             + " containers to guaranteed.");
     opts.addOption("log_properties", true, "log4j.properties file");
     opts.addOption("keep_containers_across_application_attempts", false,
-      "Flag to indicate whether to keep containers across application attempts." +
-      " If the flag is true, running containers will not be killed when" +
-      " application attempt fails and these containers will be retrieved by" +
-      " the new application attempt ");
+        "Flag to indicate whether to keep containers across application "
+            + "attempts."
+            + " If the flag is true, running containers will not be killed when"
+            + " application attempt fails and these containers will be "
+            + "retrieved by"
+            + " the new application attempt ");
     opts.addOption("attempt_failures_validity_interval", true,
       "when attempt_failures_validity_interval in milliseconds is set to > 0," +
       "the failure number will not take failures which happen out of " +
@@ -891,6 +893,10 @@ public class Client {
     }
     vargs.add("--priority " + String.valueOf(shellCmdPriority));
 
+    if (keepContainers) {
+      vargs.add("--keep_containers_across_application_attempts");
+    }
+
     for (Map.Entry<String, String> entry : shellEnv.entrySet()) {
       vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue());
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message