hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [41/51] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha
Date Thu, 22 Dec 2016 19:32:07 GMT
YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour
Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b54bf1d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b54bf1d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b54bf1d1

Branch: refs/heads/yarn-native-services
Commit: b54bf1d16032c54e2f2a53f6fe707ae4c2ccf9d1
Parents: 340967d
Author: Billie Rinaldi <billie@apache.org>
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He <jianhe@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  |  6 +-
 .../slider/client/SliderYarnClientImpl.java     | 94 +++++++++++++-------
 .../apache/slider/common/tools/SliderUtils.java | 37 +++++++-
 .../slider/core/registry/YarnAppListClient.java | 21 ++++-
 .../providers/AbstractClientProvider.java       | 28 +++++-
 .../providers/docker/DockerClientProvider.java  |  8 ++
 .../servicemonitor/YarnApplicationProbe.java    | 12 +--
 7 files changed, 160 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements
RunSe
 
     // add the tags if available
     Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
-        appOperations);
+        appOperations, clustername);
 
     Credentials credentials = null;
     if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements
RunSe
             appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
     } else {
       // scan for instance in single --state state
-      List<ApplicationReport> userInstances = yarnClient.listDeployedInstances("");
       state = state.toUpperCase(Locale.ENGLISH);
       YarnApplicationState desiredState = extractYarnApplicationState(state);
+      List<ApplicationReport> userInstances = yarnClient
+          .listDeployedInstances("", EnumSet.of(desiredState), name);
       ApplicationReport foundInstance =
           yarnClient.findAppInInstanceList(userInstances, name, desiredState);
       if (foundInstance != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
    */
   public List<ApplicationReport> listDeployedInstances(String user)
     throws YarnException, IOException {
+    return listDeployedInstances(user, null);
+  }
+
+  /**
+   * List Slider <i>deployed</i>instances belonging to a specific user in a
+   * given set of states.
+   * <p>
+   * Deployed means: known about in the YARN cluster; it will include all apps
+   * in the specified set of states.
+   *
+   * @param user
+   *          user: "" means all users
+   * @param appStates
+   *          filter by a set of YarnApplicationState
+   * @return a possibly empty list of Slider AMs
+   * @throws YarnException
+   * @throws IOException
+   */
+  public List<ApplicationReport> listDeployedInstances(String user,
+      EnumSet<YarnApplicationState> appStates)
+      throws YarnException, IOException {
+    return listDeployedInstances(user, appStates, null);
+  }
+
+  /**
+   * List Slider <i>deployed</i>instances belonging to a specific user in a
+   * given set of states and filtered by an application name tag.
+   * <p>
+   * Deployed means: known about in the YARN cluster; it will include all apps
+   * in the specified set of states and tagged with the specified app name.
+   *
+   * @param user
+   *          user: "" means all users
+   * @param appStates
+   *          filter by a set of YarnApplicationState
+   * @param appname
+   *          an application name tag in the format defined by
+   *          {@link SliderUtils#createNameTag(String)}
+   * @return a possibly empty list of Slider AMs
+   * @throws YarnException
+   * @throws IOException
+   */
+  public List<ApplicationReport> listDeployedInstances(String user,
+      EnumSet<YarnApplicationState> appStates, String appname)
+      throws YarnException, IOException {
     Preconditions.checkArgument(user != null, "Null User");
     Set<String> types = new HashSet<>(1);
     types.add(SliderKeys.APP_TYPE);
-    List<ApplicationReport> allApps = getApplications(types);
+    Set<String> tags = null;
+    if (appname != null) {
+      tags = Collections.singleton(SliderUtils.createNameTag(appname));
+    }
+    List<ApplicationReport> allApps = getApplications(types, appStates, tags);
     List<ApplicationReport> results = new ArrayList<>();
     for (ApplicationReport report : allApps) {
       if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
@@ -136,20 +187,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
    * @param appname application name
    * @return the list of all matching application instances
    */
-  public List<ApplicationReport> findAllInstances(String user,
-                                                  String appname)
+  public List<ApplicationReport> findAllInstances(String user, String appname)
       throws IOException, YarnException {
     Preconditions.checkArgument(appname != null, "Null application name");
 
-    List<ApplicationReport> instances = listDeployedInstances(user);
-    List<ApplicationReport> results =
-      new ArrayList<>(instances.size());
-    for (ApplicationReport report : instances) {
-      if (report.getName().equals(appname)) {
-        results.add(report);
-      }
-    }
-    return results;
+    return listDeployedInstances(user, null, appname);
   }
   
   /**
@@ -204,14 +246,12 @@ public class SliderYarnClientImpl extends YarnClientImpl {
       // user wants all instances killed
       String user = getUsername();
       log.info("Killing all applications belonging to {}", user);
-      Collection<ApplicationReport> instances = listDeployedInstances(user);
+      Collection<ApplicationReport> instances = listDeployedInstances(user,
+          SliderUtils.getAllLiveAppStates());
       for (ApplicationReport instance : instances) {
-        if (isApplicationLive(instance)) {
-          ApplicationId appId = instance.getApplicationId();
-          log.info("Killing Application {}", appId);
-
-          killRunningApplication(appId, "forced kill");
-        }
+        ApplicationId appId = instance.getApplicationId();
+        log.info("Killing Application {}", appId);
+        killRunningApplication(appId, "forced kill");
       }
     } else {
       ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
@@ -290,21 +330,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
    * @return the list of all matching application instances
    */
   public List<ApplicationReport> findAllLiveInstances(String user,
-                                                      String appname) throws
-                                                                      YarnException,
-                                                                      IOException {
+      String appname) throws YarnException, IOException {
     Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
         "Null/empty application name");
-    List<ApplicationReport> instances = listDeployedInstances(user);
-    List<ApplicationReport> results =
-      new ArrayList<ApplicationReport>(instances.size());
-    for (ApplicationReport app : instances) {
-      if (app.getName().equals(appname)
-          && isApplicationLive(app)) {
-        results.add(app);
-      }
-    }
-    return results;
+    return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(),
+        appname);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b101d34..4457957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -104,6 +104,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
@@ -2656,4 +2657,38 @@ public final class SliderUtils {
     }
     return prefix;
   }
-}
+
+  public static String createNameTag(String name) {
+    return "Name: " + name;
+  }
+
+  public static String createVersionTag(String version) {
+    return "Version: " + version;
+  }
+
+  public static String createDescriptionTag(String description) {
+    return "Description: " + description;
+  }
+
+  /**
+   * Get all YarnApplicationState values which signify that an application is
+   * in RUNNING or pre-RUNNING state.
+   *
+   * @return all live app states
+   */
+  public static EnumSet<YarnApplicationState> getAllLiveAppStates() {
+    return EnumSet.range(YarnApplicationState.NEW,
+        YarnApplicationState.RUNNING);
+  }
+
+  /**
+   * Get all YarnApplicationState values which signify that an application is
+   * not live, which means it is in one of the post RUNNING states.
+   *
+   * @return all non live app states
+   */
+  public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() {
+    return EnumSet.range(YarnApplicationState.FINISHED,
+        YarnApplicationState.KILLED);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 1bdfb9c..71cc193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -86,7 +86,7 @@ public class YarnAppListClient {
   public ApplicationReport findInstance(String appname) throws
                                                         YarnException,
                                                         IOException {
-    List<ApplicationReport> instances = listInstances(null);
+    List<ApplicationReport> instances = listInstances(null, appname);
     return yarnClient.findClusterInInstanceList(instances, appname);
   }
 
@@ -107,8 +107,25 @@ public class YarnAppListClient {
    */
   public List<ApplicationReport> listInstances(String user)
       throws YarnException, IOException {
+    return listInstances(user, null);
+  }
+
+  /**
+   * List all instances belonging to a specific user and a specific appname.
+   *
+   * @param user
+   *          user if not the default. null means default, "" means all users,
+   *          otherwise it is the name of a user
+   * @param appname
+   *          application name set as an application tag
+   * @return a possibly empty list of AMs
+   * @throws YarnException
+   * @throws IOException
+   */
+  public List<ApplicationReport> listInstances(String user, String appname)
+      throws YarnException, IOException {
     String listUser = user == null ? username : user;
-    return yarnClient.listDeployedInstances(listUser);
+    return yarnClient.listDeployedInstances(listUser, null, appname);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index f59c347..01444fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.conf.AggregateConf;
 import org.apache.slider.core.conf.ConfTreeOperations;
 import org.apache.slider.core.conf.MapOperations;
@@ -37,6 +38,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.HashSet;
 import java.util.Set;
 
 import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
@@ -217,11 +219,35 @@ public abstract class AbstractClientProvider extends Configured {
    * @return the set of tags.
    */
   public Set<String> getApplicationTags(SliderFileSystem fileSystem,
-                                        ConfTreeOperations appConf) throws SliderException
{
+      ConfTreeOperations appConf, String appName) throws SliderException {
     return Collections.emptySet();
   }
 
   /**
+   * Generates a fixed format of application tags given one or more of
+   * application name, version and description. This allows subsequent query for
+   * an application with a name only, version only or description only or any
+   * combination of those as filters.
+   *
+   * @param appName name of the application
+   * @param appVersion version of the application
+   * @param appDescription brief description of the application
+   * @return
+   */
+  public final Set<String> createApplicationTags(String appName,
+      String appVersion, String appDescription) {
+    Set<String> tags = new HashSet<>();
+    tags.add(SliderUtils.createNameTag(appName));
+    if (appVersion != null) {
+      tags.add(SliderUtils.createVersionTag(appVersion));
+    }
+    if (appDescription != null) {
+      tags.add(SliderUtils.createDescriptionTag(appDescription));
+    }
+    return tags;
+  }
+
+  /**
    * Process client operations for applications such as install, configure
    * @param fileSystem
    * @param registryOperations

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index d554427..86d87ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -34,6 +34,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
 
@@ -93,4 +94,11 @@ public class DockerClientProvider extends AbstractClientProvider
       }
     }
   }
+
+  @Override
+  public Set<String> getApplicationTags(SliderFileSystem fileSystem,
+      ConfTreeOperations appConf, String appName) throws SliderException {
+    return createApplicationTags(appName, null, null);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
index adf613c..92df048 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
@@ -68,23 +68,19 @@ public class YarnApplicationProbe extends Probe {
    */
   @Override
   public ProbeStatus ping(boolean livePing) {
-    
     ProbeStatus status = new ProbeStatus();
     try {
-
-      List<ApplicationReport> instances =
-        yarnClient.listDeployedInstances(username);
-      ApplicationReport instance =
-        yarnClient.findClusterInInstanceList(instances, clustername);
+      List<ApplicationReport> instances = yarnClient
+          .listDeployedInstances(username, null, clustername);
+      ApplicationReport instance = yarnClient
+          .findClusterInInstanceList(instances, clustername);
       if (null == instance) {
         throw UnknownApplicationInstanceException.unknownInstance(clustername);
       }
-
       status.succeed(this);
     } catch (Exception e) {
       status.fail(this, e);
     }
     return status;
-
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message