hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [41/58] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
Date Mon, 06 Nov 2017 22:02:15 GMT
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0

Branch: refs/heads/yarn-native-services
Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b
Parents: 8851209
Author: Jian He <jianhe@apache.org>
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Mon Nov 6 13:30:17 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  2 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   |  7 ++++
 .../hadoop-yarn-services-core/pom.xml           |  6 ----
 .../yarn/service/api/records/Artifact.java      |  8 +++++
 .../yarn/service/api/records/ConfigFile.java    |  8 +++++
 .../service/api/records/ReadinessCheck.java     |  8 +++++
 .../src/main/resources/yarn-default.xml         |  8 +++++
 .../server/resourcemanager/webapp/RMWebApp.java | 19 +++++++++++
 .../site/markdown/yarn-service/QuickStart.md    | 34 +++++++++++++++++---
 9 files changed, 89 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
       + "webapp.ui2.war-file-path";
+  public static final String YARN_API_SERVICES_ENABLE = "yarn."
+      + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
     RM_PREFIX + "resource-tracker.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..89b020d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,12 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
+
+  @Inject
+  public ApiServer(Configuration conf) {
+    super();
+  }
+
   private static final Logger LOG =
       LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
     </dependency>
 
     <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-test</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
index 4328c6f..ce062cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -23,6 +23,9 @@ import io.swagger.annotations.ApiModelProperty;
 import java.io.Serializable;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonValue;
@@ -42,6 +45,11 @@ public class Artifact implements Serializable {
 
   private String id = null;
 
+  /**
+   * Artifact Type.  DOCKER, TARBALL or SERVICE
+   **/
+  @XmlType(name = "artifact_type")
+  @XmlEnum
   public enum TypeEnum {
     DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index c1502c7..fb088e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -26,7 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
 import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
@@ -45,6 +48,11 @@ import java.util.Objects;
 public class ConfigFile implements Serializable {
   private static final long serialVersionUID = -7009402089417704612L;
 
+  /**
+   * Config Type.  XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+   **/
+  @XmlType(name = "config_type")
+  @XmlEnum
   public enum TypeEnum {
     XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
         "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 1a25a4c..b25828f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,6 +25,9 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,6 +46,11 @@ import org.apache.hadoop.classification.InterfaceStability;
 public class ReadinessCheck implements Serializable {
   private static final long serialVersionUID = -3836839816887186801L;
 
+  /**
+   * Type. HTTP and PORT
+   **/
+  @XmlType(name = "type")
+  @XmlEnum
   public enum TypeEnum {
     HTTP("HTTP"),
     PORT("PORT");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f4b2e61..e90d0f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -212,6 +212,14 @@
   </property>
 
   <property>
+    <description>
+      Enable services rest api on ResourceManager.
+    </description>
+    <name>yarn.webapp.api-service.enable</name>
+    <value>false</value>
+  </property>
+
+  <property>
     <name>yarn.resourcemanager.resource-tracker.address</name>
     <value>${yarn.resourcemanager.hostname}:8031</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 4e36665..bee9354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -22,6 +22,8 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
 
 import java.net.InetSocketAddress;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.RMHAUtils;
@@ -38,8 +40,12 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
  */
 public class RMWebApp extends WebApp implements YarnWebParams {
 
+  private static final Log LOG =
+      LogFactory.getLog(RMWebApp.class.getName());
   private final ResourceManager rm;
   private boolean standby = false;
+  private final static String APISERVER =
+      "org.apache.hadoop.yarn.service.webapp.ApiServer";
 
   public RMWebApp(ResourceManager rm) {
     this.rm = rm;
@@ -53,6 +59,19 @@ public class RMWebApp extends WebApp implements YarnWebParams {
     bind(RMWebApp.class).toInstance(this);
 
     if (rm != null) {
+      boolean enableServiceApi = rm.getConfig()
+          .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
+      if (enableServiceApi) {
+        try {
+          // Use reflection here to load ApiServer class,
+          // this is done to avoid creating cyclic dependency
+          // between maven projects.
+          Class<?> apiServer = Class.forName(APISERVER);
+          bind(apiServer);
+        } catch (ClassNotFoundException e) {
+          LOG.warn("ApiServer REST API is not activated.");
+        }
+      }
       bind(ResourceManager.class).toInstance(rm);
     }
     route("/", RmController.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 15df0cd..afcd5f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
 ```
 
 ## Manage services on YARN via REST API
-Below steps walk you through deploying services on YARN via REST API.
- Refer to [API doc](YarnServiceAPI.md)  for the detailed API specificatiosn.
-### Start API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit
their API specs via HTTP.
+
+YARN API Server REST API can be activated in two modes: embedded or standalone.
+
+### Start Embedded API-Server as part of ResourceManager
+For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
+
+```
+  <property>
+    <description>
+      Enable services rest api on ResourceManager.
+    </description>
+    <name>yarn.webapp.api-service.enable</name>
+    <value>true</value>
+  </property>
+```
+
+Services can be deployed on YARN through the ResourceManager web endpoint.
+
+### Start Standalone API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit
their service specs via HTTP.
 ```
  yarn --daemon start apiserver
  ```
 The above command starts the API Server on the localhost at port 9191 by default. 
 
+Refer to [API doc](YarnServiceAPI.md)  for the detailed API specificatiosn.
+
 ### Deploy a service
-POST the aforementioned example service definition to the api-server endpoint: 
+
+POST the aforementioned example service definition to the ResourceManager api-server endpoint:
+```
+POST  http://localhost:8088/ws/v1/services
+```
+
+Or standalone API server:
 ```
 POST  http://localhost:9191/ws/v1/services
 ```


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message