hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [02/35] hadoop git commit: YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Date Tue, 03 Oct 2017 17:06:11 GMT
YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj
Nayak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8facf1f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8facf1f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8facf1f9

Branch: refs/heads/HDFS-7240
Commit: 8facf1f976d7e12a846f12baabf54be1b7a49f9d
Parents: dd37877
Author: Sunil G <sunilg@apache.org>
Authored: Wed Sep 27 14:37:32 2017 +0530
Committer: Sunil G <sunilg@apache.org>
Committed: Wed Sep 27 14:37:32 2017 +0530

----------------------------------------------------------------------
 .../resourcemanager/webapp/DeSelectFields.java  |  11 +-
 .../resourcemanager/webapp/dao/AppInfo.java     | 108 +++++++++++++------
 .../webapp/TestRMWebServicesApps.java           |  77 ++++++++++++-
 3 files changed, 160 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
index 258bbfa..c991766 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
@@ -91,7 +91,16 @@ public class DeSelectFields {
      * <code>RESOURCE_REQUESTS</code> is the first
      * supported type from YARN-6280.
      */
-    RESOURCE_REQUESTS("resourceRequests");
+    RESOURCE_REQUESTS("resourceRequests"),
+    /**
+     * <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
+     * RESOURCE_INFO</code> are additionally supported parameters added in
+     * YARN-6871.
+     */
+    TIMEOUTS("timeouts"),
+    APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
+    AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
+    RESOURCE_INFO("resourceInfo");
 
     private final String literals;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 236c467..40d7b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -123,7 +123,7 @@ public class AppInfo {
   protected String amNodeLabelExpression;
 
   protected ResourcesInfo resourceInfo = null;
-  protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+  private AppTimeoutsInfo timeouts;
 
   public AppInfo() {
   } // JAXB needs this
@@ -247,47 +247,87 @@ public class AppInfo {
       unmanagedApplication = appSubmissionContext.getUnmanagedAM();
       appNodeLabelExpression =
           app.getApplicationSubmissionContext().getNodeLabelExpression();
-      amNodeLabelExpression = (unmanagedApplication) ? null
-          : app.getAMResourceRequests().get(0).getNodeLabelExpression();
+      /*
+       * When the deSelects parameter contains "amNodeLabelExpression", objects
+       * pertaining to the amNodeLabelExpression are not returned. By default,
+       * this is not skipped. (YARN-6871)
+       */
+      if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+        amNodeLabelExpression = (unmanagedApplication) ?
+            null :
+            app.getAMResourceRequests().get(0).getNodeLabelExpression();
+      }
+      /*
+       * When the deSelects parameter contains "appNodeLabelExpression", objects
+       * pertaining to the appNodeLabelExpression are not returned. By default,
+       * this is not skipped. (YARN-6871)
+       */
+      if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
+        appNodeLabelExpression =
+            app.getApplicationSubmissionContext().getNodeLabelExpression();
+      }
+      /*
+       * When the deSelects parameter contains "amNodeLabelExpression", objects
+       * pertaining to the amNodeLabelExpression are not returned. By default,
+       * this is not skipped. (YARN-6871)
+       */
+      if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+        amNodeLabelExpression = (unmanagedApplication) ?
+            null :
+            app.getAMResourceRequests().get(0).getNodeLabelExpression();
+      }
 
+      /*
+       * When the deSelects parameter contains "resourceInfo", ResourceInfo
+       * objects are not returned. Default behavior is no skipping. (YARN-6871)
+       */
       // Setting partition based resource usage of application
-      ResourceScheduler scheduler = rm.getRMContext().getScheduler();
-      if (scheduler instanceof CapacityScheduler) {
-        RMAppAttempt attempt = app.getCurrentAppAttempt();
-        if (null != attempt) {
-          FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
-              .getApplicationAttempt(attempt.getAppAttemptId());
-          resourceInfo = null != ficaAppAttempt
-              ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
-              : null;
+      if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
+        ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+        if (scheduler instanceof CapacityScheduler) {
+          RMAppAttempt attempt = app.getCurrentAppAttempt();
+          if (null != attempt) {
+            FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
+                .getApplicationAttempt(attempt.getAppAttemptId());
+            resourceInfo = null != ficaAppAttempt ?
+                new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
+                null;
+          }
         }
       }
 
-      Map<ApplicationTimeoutType, Long> applicationTimeouts =
-          app.getApplicationTimeouts();
-      if (applicationTimeouts.isEmpty()) {
-        // If application is not set timeout, lifetime should be sent as default
-        // with expiryTime=UNLIMITED and remainingTime=-1
-        AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
-        timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
-        timeouts.add(timeoutInfo);
-      } else {
-        for (Map.Entry<ApplicationTimeoutType, Long> entry : app
-            .getApplicationTimeouts().entrySet()) {
-          AppTimeoutInfo timeout = new AppTimeoutInfo();
-          timeout.setTimeoutType(entry.getKey());
-          long timeoutInMillis = entry.getValue().longValue();
-          timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
-          if (app.isAppInCompletedStates()) {
-            timeout.setRemainingTime(0);
-          } else {
-            timeout.setRemainingTime(Math
-                .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+      /*
+       * When the deSelects parameter contains "appTimeouts", objects pertaining
+       * to app timeouts are not returned. By default, this is not skipped.
+       * (YARN-6871)
+       */
+      if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
+        Map<ApplicationTimeoutType, Long> applicationTimeouts =
+            app.getApplicationTimeouts();
+        if (applicationTimeouts.isEmpty()) {
+          // If application is not set timeout, lifetime should be sent
+          // as default with expiryTime=UNLIMITED and remainingTime=-1
+          AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+          timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+          timeouts = new AppTimeoutsInfo();
+          timeouts.add(timeoutInfo);
+        } else {
+          for (Map.Entry<ApplicationTimeoutType, Long> entry : app
+              .getApplicationTimeouts().entrySet()) {
+            AppTimeoutInfo timeout = new AppTimeoutInfo();
+            timeout.setTimeoutType(entry.getKey());
+            long timeoutInMillis = entry.getValue().longValue();
+            timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+            if (app.isAppInCompletedStates()) {
+              timeout.setRemainingTime(0);
+            } else {
+              timeout.setRemainingTime(Math.max(
+                  (timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+            }
+            timeouts.add(timeout);
           }
-          timeouts.add(timeout);
         }
       }
-
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 07f74a3..f0704ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1113,8 +1113,83 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     JSONArray array = apps.getJSONArray("app");
     assertEquals("incorrect number of elements", 1, array.length());
     JSONObject app = array.getJSONObject(0);
-    assertTrue("resource requests shouldn't exits",
+    assertTrue("resource requests shouldn't exist",
         !app.has("resourceRequests"));
+
+    params.clear();
+    params.add("deSelects",
+        DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    app = array.getJSONObject(0);
+    assertTrue("AMNodeLabelExpression shouldn't exist",
+        !app.has("amNodeLabelExpression"));
+
+    params.clear();
+    params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    app = array.getJSONObject(0);
+    assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
+    rm.stop();
+
+    params.clear();
+    params.add("deSelects",
+        DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    app = array.getJSONObject(0);
+    assertTrue("AppNodeLabelExpression shouldn't exist",
+        !app.has("appNodeLabelExpression"));
+    rm.stop();
+
+    params.clear();
+    params
+        .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    app = array.getJSONObject(0);
+    assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
     rm.stop();
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message