Mailing-List: contact commits-help@hive.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hive-dev@hive.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: sseth@apache.org
To: commits@hive.apache.org
Message-Id: <182ce0d773894b8997712715cf7714a7@git.apache.org>
Subject: hive git commit: HIVE-16168. llap log links should use the NM nodeId
 port instead of web port. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Date: Wed, 15 Mar 2017 07:28:02 +0000 (UTC)
archived-at: Wed, 15 Mar 2017 07:28:04 -0000

Repository: hive
Updated Branches:
  refs/heads/master f99ece64a -> ddbe226ca


HIVE-16168. llap log links should use the NM nodeId port instead of web port. (Siddharth Seth, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddbe226c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddbe226c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddbe226c

Branch: refs/heads/master
Commit: ddbe226ca2622bd7bec5bcc438abed4ac5e3ee7f
Parents: f99ece6
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed Mar 15 00:27:24 2017 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Wed Mar 15 00:27:24 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/conf/HiveConf.java    |  4 ++++
 .../hadoop/hive/llap/daemon/impl/LlapDaemon.java      | 14 ++++++++++++++
 .../hive/llap/tezplugins/LlapTaskCommunicator.java    | 13 +++++++------
 3 files changed, 25 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ddbe226c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 1fb3253..bd66d89 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3052,8 +3052,12 @@ public class HiveConf extends Configuration {
     LLAP_DAEMON_QUEUE_NAME("hive.llap.daemon.queue.name", null,
         "Queue name within which the llap slider application will run." +
         " Used in LlapServiceDriver and package.py"),
+    // TODO Move the following 2 properties out of Configuration to a constant.
     LLAP_DAEMON_CONTAINER_ID("hive.llap.daemon.container.id", null,
         "ContainerId of a running LlapDaemon. Used to publish to the registry"),
+    LLAP_DAEMON_NM_ADDRESS("hive.llap.daemon.nm.address", null,
+        "NM Address host:rpcPort for the NodeManager on which the instance of the daemon is running.\n" +
+        "Published to the llap registry. Should never be set by users"),
     LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false,
       "TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"),
     LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS(

http://git-wip-us.apache.org/repos/asf/hive/blob/ddbe226c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 6a5adc9..95bc675 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -472,6 +472,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
       LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
 
       String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
+
       String appName = null;
       if (containerIdStr != null && !containerIdStr.isEmpty()) {
         daemonConf.set(ConfVars.LLAP_DAEMON_CONTAINER_ID.varname, containerIdStr);
@@ -486,6 +487,19 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
         appName = null;
       }
 
+      String nmHost = System.getenv(ApplicationConstants.Environment.NM_HOST.name());
+      String nmPort = System.getenv(ApplicationConstants.Environment.NM_PORT.name());
+      if (!org.apache.commons.lang3.StringUtils.isBlank(nmHost) && !org.apache.commons.lang3.StringUtils.isBlank(nmPort)) {
+        String nmAddress = nmHost + ":" + nmPort;
+        daemonConf.set(ConfVars.LLAP_DAEMON_NM_ADDRESS.varname, nmAddress);
+      } else {
+        daemonConf.unset(ConfVars.LLAP_DAEMON_NM_ADDRESS.varname);
+        // Unlikely, but log the actual values in case one of the two was empty/null
+        LOG.warn(
+            "NodeManager host/port not found in environment. Values retrieved: host={}, port={}",
+            nmHost, nmPort);
+      }
+
       int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
 
       String workDirsString = System.getenv(ApplicationConstants.Environment.LOCAL_DIRS.name());

http://git-wip-us.apache.org/repos/asf/hive/blob/ddbe226c/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
index 8bb6cab..18ce03c 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -14,6 +14,7 @@
 
 package org.apache.hadoop.hive.llap.tezplugins;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hive.llap.registry.ServiceInstance;
 import org.apache.hadoop.io.Writable;
 
@@ -120,7 +121,6 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
   private final String user;
   private String amHost;
   private String timelineServerUri;
-  private int nmPort;
 
   // These two structures track the list of known nodes, and the list of nodes which are sending in keep-alive heartbeats.
   // Primarily for debugging purposes a.t.m, since there's some unexplained TASK_TIMEOUTS which are currently being observed.
@@ -194,7 +194,6 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
     String scheme = WebAppUtils.getHttpSchemePrefix(conf);
     String ahsUrl = WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
     this.timelineServerUri = WebAppUtils.getURLWithScheme(scheme, ahsUrl);
-    this.nmPort = Integer.valueOf(WebAppUtils.getNMWebAppURLWithoutScheme(conf).split(":")[1]);
   }
 
   @Override
@@ -554,6 +553,7 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
         containerNodeId, e.getMessage());
       return null;
     }
+    // Once NodeId includes fragmentId - this becomes a lot more reliable.
     if (instanceSet != null) {
       ServiceInstance matchedInstance = null;
       for (ServiceInstance instance : instanceSet) {
@@ -565,8 +565,9 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
       if (matchedInstance != null) {
         String containerIdString = matchedInstance.getProperties()
           .get(HiveConf.ConfVars.LLAP_DAEMON_CONTAINER_ID.varname);
-        if (containerIdString != null) {
-          return constructLlapLogUrl(attemptID, containerIdString, isDone, containerNodeId.getHost());
+        String nmNodeAddress = matchedInstance.getProperties().get(ConfVars.LLAP_DAEMON_NM_ADDRESS.varname);
+        if (!StringUtils.isBlank(containerIdString) && !StringUtils.isBlank(nmNodeAddress)) {
+          return constructLlapLogUrl(attemptID, containerIdString, isDone, nmNodeAddress);
         }
       }
     }
@@ -574,10 +575,10 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
   }
 
   private String constructLlapLogUrl(final TezTaskAttemptID attemptID, final String containerIdString,
-    final boolean isDone, final String nmHost) {
+    final boolean isDone, final String nmAddress) {
     String dagId = attemptID.getTaskID().getVertexID().getDAGId().toString();
     String filename = JOINER.join(currentHiveQueryId, "-", dagId, ".log", (isDone ? ".done" : ""),
-      "?nm.id=", nmHost, ":", nmPort);
+      "?nm.id=", nmAddress);
     String url = PATH_JOINER.join(timelineServerUri, "ws", "v1", "applicationhistory", "containers",
       containerIdString, "logs", filename);
     return url;