hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [1/2] hive git commit: HIVE-9814 : LLAP: JMX web-service end points for monitoring & metrics (Gopal V, Sergey Shelukhin)
Date Wed, 03 Jun 2015 21:50:37 GMT
Repository: hive
Updated Branches:
  refs/heads/llap cb24a7678 -> f032e5488


HIVE-9814 : LLAP: JMX web-service end points for monitoring & metrics (Gopal V, Sergey
Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/54127078
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/54127078
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/54127078

Branch: refs/heads/llap
Commit: 54127078f3a87b9f8dc9529467a04206be0cc398
Parents: cb24a76
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Jun 3 14:25:38 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Jun 3 14:25:38 2015 -0700

----------------------------------------------------------------------
 .../llap/configuration/LlapConfiguration.java   |  6 +++
 llap-server/pom.xml                             | 11 ++++
 .../hive/llap/daemon/impl/LlapDaemon.java       |  6 +++
 .../impl/LlapDaemonProtocolServerImpl.java      |  2 +-
 .../registry/impl/LlapRegistryService.java      | 18 -------
 .../registry/impl/LlapYarnRegistryImpl.java     | 24 +++++++++
 .../llap/daemon/services/impl/LlapWebApp.java   | 12 +++++
 .../daemon/services/impl/LlapWebServices.java   | 53 ++++++++++++++++++++
 .../src/main/resources/webapps/llap/.keep       |  0
 9 files changed, 113 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
index 194deee..b57ae80 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
@@ -133,4 +133,10 @@ public class LlapConfiguration extends Configuration {
   public static final String LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS
=
       LLAP_PREFIX + "task.communicator.connection.sleep-between-retries-millis";
   public static final long LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS_DEFAULT
= 2000l;
+
+  public static final String LLAP_DAEMON_SERVICE_PORT = LLAP_DAEMON_PREFIX + "service.port";
+  public static final int LLAP_DAEMON_SERVICE_PORT_DEFAULT = 15002;
+
+  public static final String LLAP_DAEMON_SERVICE_SSL = LLAP_DAEMON_PREFIX + "service.ssl";
+  public static final boolean LLAP_DAEMON_SERVICE_SSL_DEFAULT = false;
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index 4fcd705..19f41fe 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -235,6 +235,17 @@
   <build>
     <sourceDirectory>${basedir}/src/java</sourceDirectory>
     <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+    <resources>
+    <resource>
+      <directory>src/main/resources</directory>
+      <excludes>
+        <exclude>*.py</exclude>
+        <exclude>*.pyc</exclude>
+        <exclude>llap-daemon-log4j.properties</exclude>
+      </excludes>
+      <filtering>false</filtering>
+    </resource>
+    </resources>
     <plugins>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index d55b7cf..067e213 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryComp
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.services.impl.LlapWebServices;
 import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
 import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
@@ -54,6 +55,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner,
Llap
   private final LlapDaemonProtocolServerImpl server;
   private final ContainerRunnerImpl containerRunner;
   private final LlapRegistryService registry;
+  private final LlapWebServices webServices;
   private final AtomicLong numSubmissions = new AtomicLong(0);
   private JvmPauseMonitor pauseMonitor;
   private final ObjectName llapDaemonInfoBean;
@@ -153,6 +155,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner,
Llap
         metrics);
 
     this.registry = new LlapRegistryService(true);
+    this.webServices = new LlapWebServices();
   }
 
   private void printAsciiArt() {
@@ -174,6 +177,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner,
Llap
     server.init(conf);
     containerRunner.init(conf);
     registry.init(conf);
+    webServices.init(conf);
     LlapIoProxy.setDaemon(true);
     LlapIoProxy.initializeLlapIo(conf);
   }
@@ -185,6 +189,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner,
Llap
     containerRunner.start();
     registry.start();
     registry.registerWorker();
+    webServices.start();
   }
 
   public void serviceStop() throws Exception {
@@ -194,6 +199,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner,
Llap
     server.stop();
     registry.unregisterWorker();
     registry.stop();
+    webServices.stop();
     ShuffleHandler.shutdown();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
index 8cb9715..cf3cc78 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
@@ -68,7 +68,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService
   }
 
   @Override
-  public SubmitWorkResponseProto submitWork(RpcController controller,
+  public SubmitWorkResponseProto submitWork (RpcController controller,
                                             SubmitWorkRequestProto request) throws
       ServiceException {
     try {

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
index 9f5702d..f727e09 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
@@ -14,31 +14,13 @@
 package org.apache.hadoop.hive.llap.daemon.registry.impl;
 
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.Map;
-import java.util.UUID;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.daemon.registry.ServiceInstanceSet;
 import org.apache.hadoop.hive.llap.daemon.registry.ServiceRegistry;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
-import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.log4j.Logger;
-import org.apache.zookeeper.CreateMode;
-
-import com.google.common.base.Preconditions;
 
 public class LlapRegistryService extends AbstractService {
 

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
index ace080e..cb1b1d0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
@@ -16,6 +16,9 @@ package org.apache.hadoop.hive.llap.daemon.registry.impl;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -46,6 +49,7 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.log4j.Logger;
+import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.zookeeper.CreateMode;
 
 import com.google.common.base.Preconditions;
@@ -117,6 +121,25 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
         shufflePort);
   }
 
+  public Endpoint getServicesEndpoint() {
+    final int servicePort =
+        conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT,
+            LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT);
+    final boolean isSSL =
+        conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL,
+            LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT);
+    final String scheme = isSSL ? "https" : "http";
+    final URL serviceURL;
+    try {
+      serviceURL = new URL(scheme, hostname, servicePort, "");
+      return RegistryTypeUtils.webEndpoint("services", serviceURL.toURI());
+    } catch (MalformedURLException e) {
+      throw new TezUncheckedException(e);
+    } catch (URISyntaxException e) {
+      throw new TezUncheckedException("llap service URI for " + hostname + " is invalid",
e);
+    }
+  }
+
   private final String getPath() {
     return this.path;
   }
@@ -127,6 +150,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
     ServiceRecord srv = new ServiceRecord();
     srv.addInternalEndpoint(getRpcEndpoint());
     srv.addInternalEndpoint(getShuffleEndpoint());
+    srv.addExternalEndpoint(getServicesEndpoint());
 
     for (Map.Entry<String, String> kv : this.conf) {
       if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
new file mode 100644
index 0000000..cb57c9a
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
@@ -0,0 +1,12 @@
+package org.apache.hadoop.hive.llap.daemon.services.impl;
+
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+
+public class LlapWebApp extends WebApp {
+
+  @Override
+  public void setup() {
+    // JMX / config are defaults
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
new file mode 100644
index 0000000..2275719
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
@@ -0,0 +1,53 @@
+package org.apache.hadoop.hive.llap.daemon.services.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.log4j.Logger;
+
+public class LlapWebServices extends AbstractService {
+
+  private static final Logger LOG = Logger.getLogger(LlapWebServices.class);
+
+  private int port;
+  private boolean ssl;
+  private Configuration conf;
+  private WebApp webApp;
+  private LlapWebApp webAppInstance;
+
+  public LlapWebServices() {
+    super("LlapWebServices");
+  }
+
+  @Override
+  public void serviceInit(Configuration conf) {
+
+    this.conf = new Configuration(conf);
+    this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+
+    this.port = conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT,
+            LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT);
+    this.ssl = conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL,
+            LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT);
+
+    this.webAppInstance = new LlapWebApp();
+  }
+
+  @Override
+  public void serviceStart() throws Exception {
+    String bindAddress = "0.0.0.0";
+    this.webApp =
+        WebApps.$for("llap").at(bindAddress).at(port).with(getConfig())
+        /* TODO: security negotiation here */
+            .start();
+  }
+
+  public void serviceStop() throws Exception {
+    if (this.webApp != null) {
+      this.webApp.stop();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/main/resources/webapps/llap/.keep
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/webapps/llap/.keep b/llap-server/src/main/resources/webapps/llap/.keep
new file mode 100644
index 0000000..e69de29


Mime
View raw message