hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject svn commit: r1504291 - in /hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ hadoop-yarn/hadoop-yarn-serv...
Date Wed, 17 Jul 2013 21:45:02 GMT
Author: hitesh
Date: Wed Jul 17 21:45:02 2013
New Revision: 1504291

URL: http://svn.apache.org/r1504291
Log:
Merge r1504290 from branch-2 to branch-2.1-beta for YARN-865. RM webservices can't query based
on application Types. Contributed by Xuan Gong.

Modified:
    hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
    hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
    hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1504291&r1=1504290&r2=1504291&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Wed Jul 17 21:45:02
2013
@@ -462,6 +462,9 @@ Release 2.1.0-beta - 2013-07-02
     YARN-922. Change FileSystemRMStateStore to use directories (Jian He via
     bikas)
 
+    YARN-865. RM webservices can't query based on application Types. (Xuan Gong
+    via hitesh)
+
   OPTIMIZATIONS
 
     YARN-512. Log aggregation root directory check is more expensive than it

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1504291&r1=1504290&r2=1504291&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Wed Jul 17 21:45:02 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.re
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 import javax.servlet.http.HttpServletRequest;
@@ -231,11 +233,13 @@ public class RMWebServices {
       @QueryParam("startedTimeBegin") String startedBegin,
       @QueryParam("startedTimeEnd") String startedEnd,
       @QueryParam("finishedTimeBegin") String finishBegin,
-      @QueryParam("finishedTimeEnd") String finishEnd) {
+      @QueryParam("finishedTimeEnd") String finishEnd,
+      @QueryParam("applicationTypes") Set<String> applicationTypes) {
     long num = 0;
     boolean checkCount = false;
     boolean checkStart = false;
     boolean checkEnd = false;
+    boolean checkAppTypes = false;
     long countNum = 0;
 
     // set values suitable in case both of begin/end not specified
@@ -291,6 +295,27 @@ public class RMWebServices {
           "finishTimeEnd must be greater than finishTimeBegin");
     }
 
+    Set<String> appTypes = new HashSet<String>();
+    if (!applicationTypes.isEmpty()) {
+      for (String applicationType : applicationTypes) {
+        if (applicationType != null && !applicationType.trim().isEmpty()) {
+          if (applicationType.indexOf(",") == -1) {
+            appTypes.add(applicationType.trim());
+          } else {
+            String[] types = applicationType.split(",");
+            for (String type : types) {
+              if (!type.trim().isEmpty()) {
+                appTypes.add(type.trim());
+              }
+            }
+          }
+        }
+      }
+    }
+    if (!appTypes.isEmpty()) {
+      checkAppTypes = true;
+    }
+
     final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
         .getRMApps();
     AppsInfo allApps = new AppsInfo();
@@ -332,6 +357,10 @@ public class RMWebServices {
           continue;
         }
       }
+      if (checkAppTypes
+          && !appTypes.contains(rmapp.getApplicationType())) {
+        continue;
+      }
 
       if (checkStart
           && (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd))
{

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1504291&r1=1504290&r2=1504291&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
Wed Jul 17 21:45:02 2013
@@ -638,6 +638,191 @@ public class TestRMWebServicesApps exten
   }
 
   @Test
+  public void testAppsQueryAppTypes() throws JSONException, Exception {
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    Thread.sleep(1);
+    RMApp app1 = rm.submitApp(1024);
+    amNodeManager.nodeHeartbeat(true);
+    // finish App
+    MockAM am = rm
+        .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
+    am.registerAppAttempt();
+    am.unregisterAppAttempt();
+    amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
+        1, ContainerState.COMPLETE);
+
+    rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+        .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+    rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+        .getShortUserName(), null, false, null, 2, null, "NON-YARN");
+
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("apps").queryParam("applicationTypes", "MAPREDUCE")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    JSONArray array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("MAPREDUCE",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN")
+            .queryParam("applicationTypes", "MAPREDUCE")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("MAPREDUCE")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("MAPREDUCE")));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN,NON-YARN")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("NON-YARN")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("NON-YARN")));
+
+    r = resource();
+    response = r.path("ws").path("v1").path("cluster")
+        .path("apps").queryParam("applicationTypes", "")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN,NON-YARN")
+            .queryParam("applicationTypes", "MAPREDUCE")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN")
+            .queryParam("applicationTypes", "")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("YARN",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", ",,, ,, YARN ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("YARN",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", ",,, ,,  ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("NON-YARN")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("NON-YARN")));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", " YARN, ,  ,,,")
+            .queryParam("applicationTypes", "MAPREDUCE , ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("MAPREDUCE")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("MAPREDUCE")));
+
+    rm.stop();
+  }
+
+  @Test
   public void testSingleApp() throws JSONException, Exception {
     rm.start();
     MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1504291&r1=1504290&r2=1504291&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Wed Jul 17 21:45:02 2013
@@ -1119,6 +1119,7 @@ ResourceManager REST API's.
   * startedTimeEnd - applications with start time ending with this time, specified in ms
since epoch
   * finishedTimeBegin - applications with finish time beginning with this time, specified
in ms since epoch
   * finishedTimeEnd - applications with finish time ending with this time, specified in ms
since epoch
+  * applicationTypes - applications matching the given application types, specified as a
comma-separated list.
 ------
 
 ** Elements of the <apps> (Applications) object



Mime
View raw message