hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject [4/4] hive git commit: HIVE-12952 : Show query sub-pages on webui (Szehon, reviewed by Aihua Xu and Lenni Kuff)
Date Wed, 03 Feb 2016 21:20:30 GMT
HIVE-12952 : Show query sub-pages on webui (Szehon, reviewed by Aihua Xu and Lenni Kuff)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d94c0b0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d94c0b0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d94c0b0

Branch: refs/heads/master
Commit: 2d94c0b0bd9fe88747eb749dba8e1ab51d94a4ff
Parents: f2e9edb
Author: Szehon Ho <szehon@cloudera.com>
Authored: Wed Feb 3 22:18:49 2016 +0100
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Wed Feb 3 22:18:49 2016 +0100

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   15 +
 .../apache/hadoop/hive/ql/log/PerfLogger.java   |    9 +
 .../java/org/apache/hive/http/HttpServer.java   |    2 +-
 pom.xml                                         |   12 +
 .../java/org/apache/hadoop/hive/ql/Driver.java  |   48 +-
 .../org/apache/hadoop/hive/ql/QueryDisplay.java |  253 +++
 .../apache/hadoop/hive/ql/metadata/Hive.java    |    6 +-
 service/pom.xml                                 |   51 +-
 .../org/apache/hive/tmpl/QueryProfileTmpl.jamon |  298 +++
 .../hive/service/cli/operation/Operation.java   |    4 +
 .../service/cli/operation/OperationManager.java |   82 +-
 .../service/cli/operation/SQLOperation.java     |   39 +-
 .../cli/operation/SQLOperationDisplay.java      |   99 +
 .../cli/operation/SQLOperationDisplayCache.java |   39 +
 .../service/cli/operation/SQLOperationInfo.java |   48 -
 .../apache/hive/service/server/HiveServer2.java |    2 +
 .../service/servlet/QueryProfileServlet.java    |   49 +
 .../hive-webapps/hiveserver2/hiveserver2.jsp    |   72 +-
 .../hive-webapps/static/js/bootstrap.js         | 1999 ++++++++++++++++++
 .../hive-webapps/static/js/bootstrap.min.js     |    6 +
 .../hive-webapps/static/js/jquery.min.js        |    2 +
 .../src/resources/hive-webapps/static/js/tab.js |   38 +
 22 files changed, 3028 insertions(+), 145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6678de6..73e6c21 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3741,6 +3741,21 @@ public class HiveConf extends Configuration {
     }
   }
 
+  /**
+   * @return true if HS2 webui is enabled
+   */
+  public boolean isWebUiEnabled() {
+    return this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0;
+  }
+
+  /**
+   * @return true if HS2 webui query-info cache is enabled
+   */
+  public boolean isWebUiQueryInfoCacheEnabled() {
+    return isWebUiEnabled() && this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0;
+  }
+
+
   public static boolean isLoadMetastoreConfig() {
     return loadMetastoreConfig;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index d4194cf..8fa5cbf 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.log;
 
+import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.hive.common.metrics.common.Metrics;
 import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -222,4 +223,12 @@ public class PerfLogger {
       LOG.warn("Error recording metrics", e);
     }
   }
+
+  public ImmutableMap<String, Long> getStartTimes() {
+    return ImmutableMap.copyOf(startTimes);
+  }
+
+  public ImmutableMap<String, Long> getEndTimes() {
+    return ImmutableMap.copyOf(endTimes);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hive/http/HttpServer.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java
index 9e23b11..b8836de 100644
--- a/common/src/java/org/apache/hive/http/HttpServer.java
+++ b/common/src/java/org/apache/hive/http/HttpServer.java
@@ -435,7 +435,7 @@ public class HttpServer {
    * @param pathSpec The path spec for the servlet
    * @param clazz The servlet class
    */
-  void addServlet(String name, String pathSpec,
+  public void addServlet(String name, String pathSpec,
       Class<? extends HttpServlet> clazz) {
     ServletHolder holder = new ServletHolder(clazz);
     if (name != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8c2257f..3c06188 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,6 +140,8 @@
     <!-- jackson 1 and 2 lines can coexist without issue, as they have different artifactIds -->
     <jackson.new.version>2.4.2</jackson.new.version>
     <jasper.version>5.5.23</jasper.version>
+    <jamon.plugin.version>2.3.4</jamon.plugin.version>
+    <jamon-runtime.version>2.3.1</jamon-runtime.version>
     <javaewah.version>0.3.2</javaewah.version>
     <javax-servlet.version>3.0.0.v201112011016</javax-servlet.version>
     <javolution.version>5.5.1</javolution.version>
@@ -725,6 +727,11 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>org.jamon</groupId>
+        <artifactId>jamon-runtime</artifactId>
+        <version>${jamon-runtime.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -1064,6 +1071,11 @@
 	  </excludes>
 	</configuration>
       </plugin>
+      <plugin>
+        <groupId>org.jamon</groupId>
+        <artifactId>jamon-maven-plugin</artifactId>
+        <version>${jamon.plugin.version}</version>
+      </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index bcf62a7..7147a9a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -36,6 +36,7 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.ql.exec.ConditionalTask;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
 import org.apache.hadoop.hive.ql.exec.FetchTask;
-import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -159,7 +159,7 @@ public class Driver implements CommandProcessor {
   private String operationId;
 
   // For WebUI.  Kept alive after queryPlan is freed.
-  private String savedQueryString;
+  private final QueryDisplay queryDisplay = new QueryDisplay();
 
   private boolean checkConcurrency() {
     boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
@@ -387,7 +387,6 @@ public class Driver implements CommandProcessor {
     } catch (Exception e) {
       LOG.warn("WARNING! Query command could not be redacted." + e);
     }
-    this.savedQueryString = queryStr;
 
     //holder for parent command type/string when executing reentrant queries
     QueryState queryState = new QueryState();
@@ -409,6 +408,10 @@ public class Driver implements CommandProcessor {
       conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);
     }
 
+    //save some info for webUI for use after plan is freed
+    this.queryDisplay.setQueryStr(queryStr);
+    this.queryDisplay.setQueryId(queryId);
+
     LOG.info("Compiling command(queryId=" + queryId + "): " + queryStr);
 
     SessionState.get().setupQueryCurrentTimestamp();
@@ -519,11 +522,17 @@ public class Driver implements CommandProcessor {
         }
       }
 
-      if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
+      if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT) ||
+           conf.isWebUiQueryInfoCacheEnabled()) {
         String explainOutput = getExplainOutput(sem, plan, tree);
         if (explainOutput != null) {
-          LOG.info("EXPLAIN output for queryid " + queryId + " : "
+          if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
+            LOG.info("EXPLAIN output for queryid " + queryId + " : "
               + explainOutput);
+          }
+          if (conf.isWebUiQueryInfoCacheEnabled()) {
+            queryDisplay.setExplainPlan(explainOutput);
+          }
         }
       }
       return 0;
@@ -553,18 +562,20 @@ public class Driver implements CommandProcessor {
       // since it exceeds valid range of shell return values
     } finally {
       double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.COMPILE)/1000.00;
-      dumpMetaCallTimingWithoutEx("compilation");
+      ImmutableMap<String, Long> compileHMSTimings = dumpMetaCallTimingWithoutEx("compilation");
+      queryDisplay.setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings);
       restoreSession(queryState);
       LOG.info("Completed compiling command(queryId=" + queryId + "); Time taken: " + duration + " seconds");
     }
   }
 
-  private void dumpMetaCallTimingWithoutEx(String phase) {
+  private ImmutableMap<String, Long> dumpMetaCallTimingWithoutEx(String phase) {
     try {
-      Hive.get().dumpAndClearMetaCallTiming(phase);
+      return Hive.get().dumpAndClearMetaCallTiming(phase);
     } catch (HiveException he) {
       LOG.warn("Caught exception attempting to write metadata call information " + he, he);
     }
+    return null;
   }
 
   /**
@@ -1186,6 +1197,13 @@ public class Driver implements CommandProcessor {
             + org.apache.hadoop.util.StringUtils.stringifyException(e));
       }
     }
+
+    //Save compile-time PerfLogging for WebUI.
+    //Execution-time Perf logs are done by either another thread's PerfLogger
+    //or a reset PerfLogger.
+    PerfLogger perfLogger = SessionState.getPerfLogger();
+    queryDisplay.setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes());
+    queryDisplay.setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes());
     return ret;
   }
 
@@ -1343,6 +1361,8 @@ public class Driver implements CommandProcessor {
     }
 
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN);
+    queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes());
+    queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes());
 
     // Take all the driver run hooks and post-execute them.
     try {
@@ -1414,6 +1434,7 @@ public class Driver implements CommandProcessor {
   }
 
   private CommandProcessorResponse createProcessorResponse(int ret) {
+    queryDisplay.setErrorMessage(errorMessage);
     return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError);
   }
 
@@ -1544,6 +1565,7 @@ public class Driver implements CommandProcessor {
         // Launch upto maxthreads tasks
         Task<? extends Serializable> task;
         while ((task = driverCxt.getRunnable(maxthreads)) != null) {
+          queryDisplay.addTask(task);
           TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, driverCxt);
           if (!runner.isRunning()) {
             break;
@@ -1556,6 +1578,7 @@ public class Driver implements CommandProcessor {
           continue;
         }
         hookContext.addCompleteTask(tskRun);
+        queryDisplay.setTaskCompleted(tskRun.getTask().getId(), tskRun.getTaskResult());
 
         Task<? extends Serializable> tsk = tskRun.getTask();
         TaskResult result = tskRun.getTaskResult();
@@ -1694,9 +1717,11 @@ public class Driver implements CommandProcessor {
       if (noName) {
         conf.set(MRJobConfig.JOB_NAME, "");
       }
-      dumpMetaCallTimingWithoutEx("execution");
       double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE)/1000.00;
 
+      ImmutableMap<String, Long> executionHMSTimings = dumpMetaCallTimingWithoutEx("execution");
+      queryDisplay.setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings);
+
       Map<String, MapRedStats> stats = SessionState.get().getMapRedStats();
       if (stats != null && !stats.isEmpty()) {
         long totalCpu = 0;
@@ -1952,8 +1977,8 @@ public class Driver implements CommandProcessor {
   }
 
 
-  public String getQueryString() {
-    return savedQueryString == null ? "Unknown" : savedQueryString;
+  public QueryDisplay getQueryDisplay() {
+    return queryDisplay;
   }
 
   /**
@@ -1963,5 +1988,4 @@ public class Driver implements CommandProcessor {
   public void setOperationId(String opId) {
     this.operationId = opId;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
new file mode 100644
index 0000000..c87c825
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskResult;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Some limited query information to save for WebUI.
+ *
+ * The class is synchronized, as WebUI may access information about a running query.
+ */
+public class QueryDisplay {
+
+  // Member variables
+  private String queryStr;
+  private String explainPlan;
+  private String errorMessage;
+  private String queryId;
+
+  private final Map<Phase, Map<String, Long>> hmsTimingMap = new HashMap();
+  private final Map<Phase, Map<String, Long>> perfLogStartMap = new HashMap();
+  private final Map<Phase, Map<String, Long>> perfLogEndMap = new HashMap();
+
+  private final LinkedHashMap<String, TaskInfo> tasks = new LinkedHashMap<String, TaskInfo>();
+
+  //Inner classes
+  public static enum Phase {
+    COMPILATION,
+    EXECUTION,
+  }
+
+  public static class TaskInfo {
+    private Integer returnVal;  //if set, determines that task is complete.
+    private String errorMsg;
+    private long endTime;
+
+    final long beginTime;
+    final String taskId;
+    final StageType taskType;
+    final String name;
+    final boolean requireLock;
+    final boolean retryIfFail;
+
+    public TaskInfo (Task task) {
+      beginTime = System.currentTimeMillis();
+      taskId = task.getId();
+      taskType = task.getType();
+      name = task.getName();
+      requireLock = task.requireLock();
+      retryIfFail = task.ifRetryCmdWhenFail();
+    }
+
+    public synchronized String getStatus() {
+      if (returnVal == null) {
+        return "Running";
+      } else if (returnVal == 0) {
+        return "Success, ReturnVal 0";
+      } else {
+        return "Failure, ReturnVal " + String.valueOf(returnVal);
+      }
+    }
+
+    public synchronized long getElapsedTime() {
+      if (endTime == 0) {
+        return System.currentTimeMillis() - beginTime;
+      } else {
+        return endTime - beginTime;
+      }
+    }
+
+    public synchronized String getErrorMsg() {
+      return errorMsg;
+    }
+
+    public synchronized long getEndTime() {
+      return endTime;
+    }
+
+    //Following methods do not need to be synchronized, because they are final fields.
+    public long getBeginTime() {
+      return beginTime;
+    }
+
+    public String getTaskId() {
+      return taskId;
+    }
+
+    public StageType getTaskType() {
+      return taskType;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public boolean isRequireLock() {
+      return requireLock;
+    }
+
+    public boolean isRetryIfFail() {
+      return retryIfFail;
+    }
+  }
+
+  public synchronized void addTask(Task task) {
+    tasks.put(task.getId(), new TaskInfo(task));
+  }
+
+  public synchronized void setTaskCompleted(String taskId, TaskResult result) {
+    TaskInfo taskInfo = tasks.get(taskId);
+    if (taskInfo != null) {
+      taskInfo.returnVal = result.getExitVal();
+      if (result.getTaskError() != null) {
+        taskInfo.errorMsg = result.getTaskError().toString();
+      }
+      taskInfo.endTime = System.currentTimeMillis();
+    }
+  }
+
+  public synchronized List<TaskInfo> getTaskInfos() {
+    List<TaskInfo> taskInfos = new ArrayList<TaskInfo>();
+    taskInfos.addAll(tasks.values());
+    return taskInfos;
+  }
+
+  public synchronized void setQueryStr(String queryStr) {
+    this.queryStr = queryStr;
+  }
+
+  public synchronized String getQueryString() {
+    return returnStringOrUnknown(queryStr);
+  }
+
+  public synchronized String getExplainPlan() {
+    return returnStringOrUnknown(explainPlan);
+  }
+
+  public synchronized void setExplainPlan(String explainPlan) {
+    this.explainPlan = explainPlan;
+  }
+
+  /**
+   * @param phase phase of query
+   * @return map of HMS Client method-calls and duration in miliseconds, during given phase.
+   */
+  public synchronized Map<String, Long> getHmsTimings(Phase phase) {
+    return hmsTimingMap.get(phase);
+  }
+
+  /**
+   * @param phase phase of query
+   * @param hmsTimings map of HMS Client method-calls and duration in miliseconds, during given phase.
+   */
+  public synchronized void setHmsTimings(Phase phase, ImmutableMap<String, Long> hmsTimings) {
+    hmsTimingMap.put(phase, hmsTimings);
+  }
+
+  /**
+   * @param phase phase of query
+   * @return map of PerfLogger call-trace name and start time in miliseconds, during given phase.
+   */
+  public synchronized Map<String, Long> getPerfLogStarts(Phase phase) {
+    return perfLogStartMap.get(phase);
+  }
+
+  /**
+   * @param phase phase of query
+   * @param perfLogStarts map of PerfLogger call-trace name and start time in miliseconds, during given phase.
+   */
+  public synchronized void setPerfLogStarts(Phase phase, ImmutableMap<String, Long> perfLogStarts) {
+    perfLogStartMap.put(phase, perfLogStarts);
+  }
+
+  /**
+   * @param phase phase of query
+   * @return map of PerfLogger call-trace name and end time in miliseconds, during given phase.
+   */
+  public synchronized Map<String, Long> getPerfLogEnds(Phase phase) {
+    return perfLogEndMap.get(phase);
+  }
+
+  /**
+   * @param phase phase of query
+   * @param perfLogEnds map of PerfLogger call-trace name and end time in miliseconds, during given phase.
+   */
+   public synchronized void setPerfLogEnds(Phase phase, ImmutableMap<String, Long> perfLogEnds) {
+    perfLogEndMap.put(phase, perfLogEnds);
+  }
+
+  /**
+   * @param phase phase of query
+   * @return map of PerfLogger call-trace name and duration in miliseconds, during given phase.
+   */
+  public synchronized Map<String, Long> getPerfLogTimes(Phase phase) {
+    Map<String, Long> times = new HashMap<>();
+    Map<String, Long> startTimes = perfLogStartMap.get(phase);
+    Map<String, Long> endTimes = perfLogEndMap.get(phase);
+    if (endTimes != null && startTimes != null) {
+      for (String timeKey : endTimes.keySet()) {
+        Long endTime = endTimes.get(timeKey);
+        Long startTime = startTimes.get(timeKey);
+        if (startTime != null) {
+          times.put(timeKey, endTime - startTime);
+        }
+      }
+    }
+    return times;
+  }
+
+  public synchronized String getErrorMessage() {
+    return errorMessage;
+  }
+
+  public synchronized void setErrorMessage(String errorMessage) {
+    this.errorMessage = errorMessage;
+  }
+
+  public synchronized String getQueryId() {
+    return returnStringOrUnknown(queryId);
+  }
+
+  public synchronized void setQueryId(String queryId) {
+    this.queryId = queryId;
+  }
+
+  private String returnStringOrUnknown(String s) {
+    return s == null ? "UNKNOWN" : s;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 0bab769..4e574f7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -44,6 +44,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -3400,7 +3401,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
     metaCallTimeMap.clear();
   }
 
-  public void dumpAndClearMetaCallTiming(String phase) {
+  public ImmutableMap<String, Long> dumpAndClearMetaCallTiming(String phase) {
     boolean phaseInfoLogged = false;
     if (LOG.isDebugEnabled()) {
       phaseInfoLogged = logDumpPhase(phase);
@@ -3420,7 +3421,10 @@ private void constructOneLBLocationMap(FileStatus fSta,
         }
       }
     }
+
+    ImmutableMap<String, Long> result = ImmutableMap.copyOf(metaCallTimeMap);
     metaCallTimeMap.clear();
+    return result;
   }
 
   private boolean logDumpPhase(String phase) {

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/pom.xml
----------------------------------------------------------------------
diff --git a/service/pom.xml b/service/pom.xml
index b2e3a84..e3f61d0 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -128,7 +128,13 @@
       <version>${hadoop.version}</version>
       <optional>true</optional>
     </dependency>
-     <!-- intra-project -->
+    <dependency>
+      <groupId>org.jamon</groupId>
+      <artifactId>jamon-runtime</artifactId>
+      <version>${jamon-runtime.version}</version>
+    </dependency>
+
+      <!-- intra-project -->
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
@@ -219,7 +225,48 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
+        <plugin>
+            <groupId>org.jamon</groupId>
+            <artifactId>jamon-maven-plugin</artifactId>
+            <executions>
+                <execution>
+                    <phase>generate-sources</phase>
+                    <goals>
+                        <goal>translate</goal>
+                    </goals>
+                    <configuration>
+                        <templateSourceDir>src/jamon</templateSourceDir>
+                        <templateOutputDir>target/generated-jamon</templateOutputDir>
+                    </configuration>
+                </execution>
+            </executions>
+        </plugin>
+        <!-- General plugins -->
+        <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-eclipse-plugin</artifactId>
+            <configuration>
+                <additionalProjectnatures>
+                    <projectnature>org.jamon.project.jamonnature</projectnature>
+                </additionalProjectnatures>
+                <buildcommands>
+                    <buildcommand>org.jamon.project.templateBuilder</buildcommand>
+                    <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand>
+                    <buildcommand>org.jamon.project.markerUpdater</buildcommand>
+                </buildcommands>
+                <additionalConfig>
+                    <file>
+                        <name>.settings/org.jamon.prefs</name>
+                        <content># now
+                            eclipse.preferences.version=1
+                            templateSourceDir=src/main/jamon
+                            templateOutputDir=target/generated-jamon
+                        </content>
+                    </file>
+                </additionalConfig>
+            </configuration>
+        </plugin>
+        <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
         <executions>

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon
----------------------------------------------------------------------
diff --git a/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon
new file mode 100644
index 0000000..c513689
--- /dev/null
+++ b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon
@@ -0,0 +1,298 @@
+<%doc>
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+</%doc>
+<%args>
+SQLOperationDisplay sod;
+</%args>
+<%import>
+java.util.*;
+org.apache.hadoop.hive.ql.QueryDisplay;
+org.apache.hive.service.cli.operation.SQLOperationDisplay;
+</%import>
+<!--[if IE]>
+<!DOCTYPE html>
+<![endif]-->
+<?xml version="1.0" encoding="UTF-8" ?>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>HiveServer2</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+
+    <link href="/static/css/bootstrap.min.css" rel="stylesheet">
+    <link href="/static/css/bootstrap-theme.min.css" rel="stylesheet">
+    <link href="/static/css/hive.css" rel="stylesheet">
+  </head>
+
+  <body>
+  <div class="navbar  navbar-fixed-top navbar-default">
+      <div class="container">
+          <div class="navbar-header">
+              <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                  <span class="icon-bar"></span>
+                  <span class="icon-bar"></span>
+                  <span class="icon-bar"></span>
+              </button>
+              <a class="navbar-brand" href="/hiveserver2.jsp"><img src="/static/hive_logo.jpeg" alt="Hive Logo"/></a>
+          </div>
+          <div class="collapse navbar-collapse">
+              <ul class="nav navbar-nav">
+                <li class="active"><a href="/">Home</a></li>
+                <li><a href="/logs/">Local logs</a></li>
+                <li><a href="/jmx">Metrics Dump</a></li>
+                <li><a href="/conf">Hive Configuration</a></li>
+                <li><a href="/stacks">Stack Trace</a></li>
+            </ul>
+          </div><!--/.nav-collapse -->
+        </div>
+      </div>
+    </div>
+
+
+    <%if sod == null %>
+         <div class="jumbotron">
+           <p>Query not found.  It may have been deleted, increase <i>hive.server2.webui.max.historic.queries</i>
+              to retain more historic query information.</p>
+         </div>
+    <%else>
+
+
+    <div class="container">
+      <div class="row inner_header">
+        <div class="page-header">
+          <h1>Query Information: <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %></h1>
+        </div>
+      </div>
+      <div class="row">
+
+
+      <div class="tabbable">
+          <ul class="nav nav-pills">
+              <li class="active"><a href="#tab_baseProfile" data-toggle="tab">Base Profile</a></li>
+              <li class=""><a href="#tab_stages" data-toggle="tab">Stages</a></li>
+              <li class=""><a href="#tab_queryPlan" data-toggle="tab">Query Plan</a></li>
+              <li class=""><a href="#tab_perfLogging" data-toggle="tab">Performance Logging</a></li>
+          </ul>
+          <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;">
+              <div class="tab-pane active" id="tab_baseProfile">
+                  <& baseProfile; sod = sod &>
+              </div>
+              <div class="tab-pane" id="tab_stages">
+                  <& stages; sod = sod &>
+              </div>
+              <div class="tab-pane" id="tab_queryPlan">
+                  <& queryPlan; sod = sod &>
+              </div>
+              <div class="tab-pane" id="tab_perfLogging">
+                  <& perfLogging; sod = sod &>
+              </div>
+          </div>
+      </div>
+
+   </%if>
+
+</div>
+</div>
+<script src="/static/js/jquery.min.js" type="text/javascript"></script>
+<script src="/static/js/bootstrap.min.js" type="text/javascript"></script>
+<script src="/static/js/tab.js" type="text/javascript"></script>
+</body>
+</html>
+
+<%def baseProfile>
+<%args>
+    SQLOperationDisplay sod;
+</%args>
+    <table class="table table-striped">
+        <tr>
+            <td>User Name</td>
+            <td><% sod.getUserName() %></td>
+        </tr>
+        <tr>
+            <td>Query String</td>
+            <td><% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %></td>
+        </tr>
+        <tr>
+            <td>Query Id</td>
+            <td><% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryId() %></td>
+        </tr>
+        <tr>
+            <td>Execution Engine</td>
+            <td><% sod.getExecutionEngine() %>
+        </tr>
+        <tr>
+            <td>State</td>
+            <td><% sod.getState() %></td>
+        </tr>
+        <tr>
+            <td>Begin Time</td>
+            <td><% new Date(sod.getBeginTime()) %></td>
+        </tr>
+        <tr>
+            <td>Elapsed Time (s)</td>
+            <td><% sod.getElapsedTime()/1000 %></td>
+        </tr>
+        <tr>
+            <td>End Time</td>
+            <td><% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %></td>
+        </tr>
+        <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getErrorMessage() != null %>
+            <tr>
+                <td>Error</td>
+                <td><% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %></td>
+            </tr>
+        </%if>
+    </table>
+</%def>
+
+<%def stages>
+<%args>
+    SQLOperationDisplay sod;
+</%args>
+   <table id="attributes_table" class="table table-striped">
+       <tr>
+           <th>Stage Id</th>
+           <th>Status</th>
+           <th>Begin Time</th>
+           <th>End Time</th>
+           <th>Elapsed Time (s)</th>
+           <th>Requires Lock</th>
+           <th>Retry If Fail</th>
+        </tr>
+
+       <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getTaskInfos() != null %>
+           <%for QueryDisplay.TaskInfo taskInfo : sod.getQueryDisplay().getTaskInfos() %>
+               <tr>
+                   <td><% taskInfo.getTaskId() + ":" + taskInfo.getTaskType() %></td>
+                   <td><% taskInfo.getStatus() %></td>
+                   <td><% new Date(taskInfo.getBeginTime()) %>
+                   <td><% taskInfo.getEndTime() == 0 ? "" : new Date(taskInfo.getEndTime()) %></td>
+                   <td><% taskInfo.getElapsedTime()/1000 %> (s) </td>
+                   <td><% taskInfo.isRequireLock() %></td>
+                   <td><% taskInfo.isRetryIfFail() %></td>
+               </tr>
+           </%for>
+       </%if>
+   </table>
+</%def>
+
+
+<%def queryPlan>
+<%args>
+    SQLOperationDisplay sod;
+</%args>
+    <div class="panel panel-default">
+      <div class="panel-heading">Explain plan</div>
+      <div class="panel-body">
+        <pre>
+        <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getExplainPlan() %>
+        </pre>
+      </div>
+    </div>
+</%def>
+
+
+<%def perfLogging>
+<%args>
+    SQLOperationDisplay sod;
+</%args>
+    <section>
+      <h3>Compile-time metadata operations</h3>
+        <table id="attributes_table" class="table table-striped">
+          <tr>
+             <th>Call Name</th>
+             <th>Time (ms)</th>
+          </tr>
+
+          <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION) != null %>
+             <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION).entrySet() %>
+                 <tr>
+                     <td><% time.getKey() %></td>
+                     <td><% time.getValue() %></td>
+                 </tr>
+             </%for>
+         </%if>
+        </table>
+      </section>
+
+    <section>
+      <h3>Execution-time metadata operations</h3>
+        <table id="attributes_table" class="table table-striped">
+          <tr>
+             <th>Call Name</th>
+             <th>Time (ms)</th>
+          </tr>
+
+          <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION) != null %>
+             <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION).entrySet() %>
+                 <tr>
+                     <td><% time.getKey() %></td>
+                     <td><% time.getValue() %></td>
+                 </tr>
+             </%for>
+         </%if>
+        </table>
+      </section>
+
+    <section>
+      <h3>Compile-Time Perf-Logger</h3>
+        <table id="attributes_table" class="table table-striped">
+          <tr>
+             <th>Compile-time Call Name</th>
+             <th>Time (ms)</th>
+          </tr>
+
+          <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION) != null %>
+             <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION).entrySet()  %>
+                 <tr>
+                     <td><% time.getKey() %></td>
+                     <td><% time.getValue() %></td>
+                 </tr>
+             </%for>
+         </%if>
+        </table>
+      </section>
+
+    <section>
+      <h3>Execution-Time Perf-Logger</h3>
+        <table id="attributes_table" class="table table-striped">
+          <tr>
+             <th>Execution-time Call Name</th>
+             <th>Time (ms)</th>
+          </tr>
+
+          <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION) != null %>
+             <%for Map.Entry<String, Long> time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION).entrySet()  %>
+                 <tr>
+                     <td><% time.getKey() %></td>
+                     <td><% time.getValue() %></td>
+                 </tr>
+             </%for>
+         </%if>
+        </table>
+      </section>
+</%def>
+
+</div>
+</div>
+<script src="/static/js/jquery.min.js" type="text/javascript"></script>
+<script src="/static/js/bootstrap.min.js" type="text/javascript"></script>
+<script src="/static/js/tab.js" type="text/javascript"></script>
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index 0c263cf..8340202 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -157,6 +157,7 @@ public abstract class Operation {
     state.validateTransition(newState);
     this.state = newState;
     setMetrics(state);
+    onNewState(state);
     this.lastAccessTime = System.currentTimeMillis();
     return this.state;
   }
@@ -420,4 +421,7 @@ public abstract class Operation {
   protected OperationState getState() {
     return state;
   }
+
+  protected void onNewState(OperationState state) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index f1ce6f6..38f27ef 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -22,7 +22,9 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -65,8 +67,10 @@ public class OperationManager extends AbstractService {
   private final ConcurrentHashMap<OperationHandle, Operation> handleToOperation =
       new ConcurrentHashMap<OperationHandle, Operation>();
 
-  //for displaying historical queries on WebUI
-  private Queue<SQLOperationInfo> historicSqlOperations;
+  //Following fields for displaying queries on WebUI
+  private Object webuiLock = new Object();
+  private SQLOperationDisplayCache historicSqlOperations;
+  private Map<String, SQLOperationDisplay> liveSqlOperations = new LinkedHashMap<String, SQLOperationDisplay>();
 
   public OperationManager() {
     super(OperationManager.class.getSimpleName());
@@ -80,9 +84,8 @@ public class OperationManager extends AbstractService {
     } else {
       LOG.debug("Operation level logging is turned off");
     }
-    if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
-      hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) {
-      historicSqlOperations = EvictingQueue.create(
+    if (hiveConf.isWebUiQueryInfoCacheEnabled()) {
+      historicSqlOperations = new SQLOperationDisplayCache(
         hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES));
     }
     super.init(hiveConf);
@@ -186,7 +189,11 @@ public class OperationManager extends AbstractService {
     Operation operation = handleToOperation.get(operationHandle);
     if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
       handleToOperation.remove(operationHandle, operation);
-      cacheOldOperationInfo(operation);
+      synchronized (webuiLock) {
+        String opKey = operationHandle.getHandleIdentifier().toString();
+        SQLOperationDisplay display = liveSqlOperations.remove(opKey);
+        historicSqlOperations.put(opKey, display);
+      }
       return operation;
     }
     return null;
@@ -194,11 +201,21 @@ public class OperationManager extends AbstractService {
 
   private void addOperation(Operation operation) {
     handleToOperation.put(operation.getHandle(), operation);
+    if (operation instanceof SQLOperation) {
+      synchronized (webuiLock) {
+        liveSqlOperations.put(operation.getHandle().getHandleIdentifier().toString(),
+          ((SQLOperation) operation).getSQLOperationDisplay());
+      }
+    }
   }
 
   private Operation removeOperation(OperationHandle opHandle) {
     Operation result = handleToOperation.remove(opHandle);
-    cacheOldOperationInfo(result);
+    synchronized (webuiLock) {
+      String opKey = opHandle.getHandleIdentifier().toString();
+      SQLOperationDisplay display = liveSqlOperations.remove(opKey);
+      historicSqlOperations.put(opKey, display);
+    }
     return result;
   }
 
@@ -329,35 +346,40 @@ public class OperationManager extends AbstractService {
     return removed;
   }
 
-  //Cache a number of historical operation info, at max number of
-  //hive.server2.webui.max.historic.queries.
-  private void cacheOldOperationInfo(Operation oldOperation) {
-    if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
-      getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) {
-      if (oldOperation instanceof SQLOperation) {
-        SQLOperation query = (SQLOperation) oldOperation;
-        SQLOperationInfo queryInfo = query.getSQLOperationInfo();
-        if (queryInfo != null) {
-          synchronized (historicSqlOperations) {
-            historicSqlOperations.add(queryInfo);
-          }
-        }
-      }
+  /**
+   * @return displays representing a number of historical SQLOperations, at max number of
+   * hive.server2.webui.max.historic.queries
+   */
+  public List<SQLOperationDisplay> getHistoricalSQLOperations() {
+    List<SQLOperationDisplay> result = new LinkedList<>();
+    synchronized (webuiLock) {
+      result.addAll(historicSqlOperations.values());
     }
+    return result;
   }
 
   /**
-   * @return a number of historical SQLOperation info, at max number of
-   * hive.server2.webui.max.historic.queries
+   * @return displays representing live SQLOperations
    */
-  public List<SQLOperationInfo> getHistoricalSQLOpInfo() {
-    List<SQLOperationInfo> result = new LinkedList<>();
-    synchronized (historicSqlOperations) {
-      Iterator<SQLOperationInfo> opIterator = historicSqlOperations.iterator();
-      while (opIterator.hasNext()) {
-        result.add(opIterator.next());
-      }
+  public List<SQLOperationDisplay> getLiveSqlOperations() {
+    List<SQLOperationDisplay> result = new LinkedList<>();
+    synchronized (webuiLock) {
+      result.addAll(liveSqlOperations.values());
     }
     return result;
   }
+
+  /**
+   * @param handle handle of SQLOperation.
+   * @return display representing a particular SQLOperation.
+   */
+  public SQLOperationDisplay getSQLOperationDisplay(String handle) {
+    synchronized (webuiLock) {
+      SQLOperationDisplay result = liveSqlOperations.get(handle);
+      if (result != null) {
+        return result;
+      }
+      return historicSqlOperations.get(handle);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index 01b1d3d..3fbbb70 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -34,8 +34,6 @@ import java.util.concurrent.RejectedExecutionException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.CharEncoding;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveVariableSource;
-import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
@@ -79,11 +77,20 @@ public class SQLOperation extends ExecuteStatementOperation {
   private SerDe serde = null;
   private boolean fetchStarted = false;
 
+  //Display for WebUI.
+  private SQLOperationDisplay sqlOpDisplay;
+
+
   public SQLOperation(HiveSession parentSession, String statement, Map<String,
       String> confOverlay, boolean runInBackground) {
     // TODO: call setRemoteUser in ExecuteStatementOperation or higher.
     super(parentSession, statement, confOverlay, runInBackground);
     setupSessionIO(parentSession.getSessionState());
+    try {
+      sqlOpDisplay = new SQLOperationDisplay(this);
+    } catch (HiveSQLException e) {
+      LOG.warn("Error calcluating SQL Operation Display for webui", e);
+    }
   }
 
   private void setupSessionIO(SessionState sessionState) {
@@ -111,6 +118,7 @@ public class SQLOperation extends ExecuteStatementOperation {
 
     try {
       driver = new Driver(sqlOperationConf, getParentSession().getUserName());
+      sqlOpDisplay.setQueryDisplay(driver.getQueryDisplay());
 
       // set the operation handle information in Driver, so that thrift API users
       // can use the operation handle they receive, to lookup query information in
@@ -162,10 +170,6 @@ public class SQLOperation extends ExecuteStatementOperation {
     }
   }
 
-  public String getQueryStr() {
-    return driver == null ? "Unknown" : driver.getQueryString();
-  }
-
   private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
     try {
       // In Hive server mode, we are not able to retry in the FetchTask
@@ -485,18 +489,17 @@ public class SQLOperation extends ExecuteStatementOperation {
   /**
    * Get summary information of this SQLOperation for display in WebUI.
    */
-  public SQLOperationInfo getSQLOperationInfo() {
-    try {
-      return new SQLOperationInfo(
-        getParentSession().getUserName(),
-        driver.getQueryString(),
-        getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE),
-        getState(),
-        (int) (System.currentTimeMillis() - getBeginTime()) / 1000,
-        System.currentTimeMillis());
-    } catch (HiveSQLException e) {
-      LOG.warn("Error calcluating SQL Operation Info for webui", e);
+  public SQLOperationDisplay getSQLOperationDisplay() {
+    return sqlOpDisplay;
+  }
+
+  @Override
+  protected void onNewState(OperationState state) {
+    if (state == OperationState.CLOSED) {
+      sqlOpDisplay.closed();
+    } else {
+      //CLOSED state not interesting, state before (FINISHED, ERROR) is.
+      sqlOpDisplay.updateState(state);
     }
-    return null;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java
new file mode 100644
index 0000000..d2ca1e7
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.operation;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryDisplay;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationState;
+
+/**
+ * Used to display some info in the HS2 WebUI.
+ *
+ * The class is synchronized, as WebUI may access information about a running query.
+ */
+public class SQLOperationDisplay {
+  public final String userName;
+  public final String executionEngine;
+  public final long beginTime;
+  public final String operationId;
+
+  public Long endTime;
+  public OperationState state;
+  public QueryDisplay queryDisplay;
+
+  public SQLOperationDisplay(SQLOperation sqlOperation) throws HiveSQLException {
+    this.state = sqlOperation.getState();
+    this.userName = sqlOperation.getParentSession().getUserName();
+    this.executionEngine = sqlOperation.getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
+    this.beginTime = System.currentTimeMillis();
+    this.operationId = sqlOperation.getHandle().getHandleIdentifier().toString();
+  }
+
+  public synchronized long getElapsedTime() {
+    if (isRunning()) {
+      return System.currentTimeMillis() - beginTime;
+    } else {
+      return endTime - beginTime;
+    }
+  }
+
+  public synchronized boolean isRunning() {
+    return endTime == null;
+  }
+
+  public synchronized QueryDisplay getQueryDisplay() {
+    return queryDisplay;
+  }
+
+  public synchronized void setQueryDisplay(QueryDisplay queryDisplay) {
+    this.queryDisplay = queryDisplay;
+  }
+
+  public String getUserName() {
+    return userName;
+  }
+
+  public String getExecutionEngine() {
+    return executionEngine;
+  }
+
+  public synchronized OperationState getState() {
+    return state;
+  }
+
+  public long getBeginTime() {
+    return beginTime;
+  }
+
+  public synchronized Long getEndTime() {
+    return endTime;
+  }
+
+  public synchronized void updateState(OperationState state) {
+    this.state = state;
+  }
+
+  public String getOperationId() {
+    return operationId;
+  }
+
+  public synchronized void closed() {
+    this.endTime = System.currentTimeMillis();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java
new file mode 100644
index 0000000..4a33d37
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.operation;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Cache some SQLOperation information for WebUI
+ */
+public class SQLOperationDisplayCache extends LinkedHashMap<String, SQLOperationDisplay> {
+
+  private final int capacity;
+
+  public SQLOperationDisplayCache(int capacity) {
+      super(capacity + 1, 1.1f, false);
+      this.capacity = capacity;
+  }
+
+  @Override
+  protected boolean removeEldestEntry(Map.Entry eldest) {
+    return size() > capacity;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
deleted file mode 100644
index 179f6dd..0000000
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.service.cli.operation;
-
-import org.apache.hive.service.cli.OperationState;
-
-/**
- * Used to display some info in the HS2 WebUI.
- */
-public class SQLOperationInfo {
-  public String userName;
-  public String queryStr;
-  public String executionEngine;
-  public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR)
-  public int elapsedTime;
-  public long endTime;
-
-  public SQLOperationInfo(
-    String userName,
-    String queryStr,
-    String executionEngine,
-    OperationState endState,
-    int elapsedTime,
-    long endTime
-  ) {
-    this.userName = userName;
-    this.queryStr = queryStr;
-    this.executionEngine = executionEngine;
-    this.endState = endState;
-    this.elapsedTime = elapsedTime;
-    this.endTime = endTime;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 958458f..c601614 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -64,6 +64,7 @@ import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
 import org.apache.hive.service.cli.thrift.ThriftCLIService;
 import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
+import org.apache.hive.service.servlet.QueryProfileServlet;
 import org.apache.logging.log4j.util.Strings;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -180,6 +181,7 @@ public class HiveServer2 extends CompositeService {
             builder.setUseSPNEGO(true);
           }
           webServer = builder.build();
+          webServer.addServlet("query_page", "/query_page", QueryProfileServlet.class);
         }
       }
     } catch (IOException ie) {

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java
new file mode 100644
index 0000000..74a374d
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.servlet;
+
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.cli.operation.SQLOperationDisplay;
+import org.apache.hive.service.cli.session.SessionManager;
+import org.apache.hive.tmpl.QueryProfileTmpl;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ * Renders a query page
+ */
+public class QueryProfileServlet extends HttpServlet {
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+    throws ServletException, IOException {
+    String opId = (String) request.getParameter("operationId");
+    ServletContext ctx = getServletContext();
+    SessionManager sessionManager =
+      (SessionManager)ctx.getAttribute("hive.sm");
+    OperationManager opManager = sessionManager.getOperationManager();
+    SQLOperationDisplay sod = opManager.getSQLOperationDisplay(opId);
+
+    new QueryProfileTmpl().render(response.getWriter(), sod);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
----------------------------------------------------------------------
diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
index a0b5d2e..8b46550 100644
--- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
+++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
@@ -24,7 +24,7 @@
   import="org.apache.hive.common.util.HiveVersionInfo"
   import="org.apache.hive.service.cli.operation.Operation"
   import="org.apache.hive.service.cli.operation.SQLOperation"
-  import="org.apache.hive.service.cli.operation.SQLOperationInfo"
+  import="org.apache.hive.service.cli.operation.SQLOperationDisplay"
   import="org.apache.hive.service.cli.session.SessionManager"
   import="org.apache.hive.service.cli.session.HiveSession"
   import="javax.servlet.ServletContext"
@@ -132,29 +132,32 @@ for (HiveSession hiveSession: hiveSessions) {
         <th>Query</th>
         <th>Execution Engine</th>
         <th>State</th>
+        <th>Begin Time</th>
         <th>Elapsed Time (s)</th>
+        <th>Drilldown Link</th>
     </tr>
-<%
-int queries = 0;
-Collection<Operation> operations = sessionManager.getOperations();
-for (Operation operation: operations) {
-  if (operation instanceof SQLOperation) {
-    SQLOperation query = (SQLOperation) operation;
-    queries++;
-%>
+    <%
+      int queries = 0;
+      Collection<SQLOperationDisplay> operations = sessionManager.getOperationManager().getLiveSqlOperations();
+      for (SQLOperationDisplay operation : operations) {
+          queries++;
+    %>
     <tr>
-        <td><%= query.getParentSession().getUserName() %></td>
-        <td><%= query.getQueryStr() %></td>
-        <td><%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %>
-        <td><%= query.getStatus().getState() %></td>
-        <td><%= (currentTime - query.getBeginTime())/1000 %></td>
+        <td><%= operation.getUserName() %></td>
+        <td><%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %></td>
+        <td><%= operation.getExecutionEngine() %>
+        <td><%= operation.getState() %></td>
+        <td><%= new Date(operation.getBeginTime()) %></td>
+        <td><%= operation.getElapsedTime()/1000 %></td>
+        <% String link = "/query_page?operationId=" + operation.getOperationId(); %>
+        <td>  <a href= <%= link %>>Query Drilldown</a> </td>
     </tr>
+
 <%
   }
-}
 %>
 <tr>
-  <td colspan="5">Total number of queries: <%= queries %></td>
+  <td colspan="7">Total number of queries: <%= queries %></td>
 </tr>
 </table>
 </section>
@@ -169,33 +172,36 @@ for (Operation operation: operations) {
         <th>Execution Engine</th>
         <th>State</th>
         <th>Elapsed Time (s)</th>
-        <th>End Time </th>
+        <th>End Time</th>
+        <th>Drilldown Link</th>
     </tr>
-<%
-queries = 0;
-List<SQLOperationInfo> sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo();
-for (SQLOperationInfo sqlOperation: sqlOperations) {
-  queries++;
-%>
+    <%
+      queries = 0;
+      operations = sessionManager.getOperationManager().getHistoricalSQLOperations();
+      for (SQLOperationDisplay operation : operations) {
+          queries++;
+    %>
     <tr>
-        <td><%= sqlOperation.userName %></td>
-        <td><%= sqlOperation.queryStr %></td>
-        <td><%= sqlOperation.executionEngine %></td>
-        <td><%= sqlOperation.endState %></td>
-        <td><%= sqlOperation.elapsedTime %></td>
-        <td><%= new Date(sqlOperation.endTime) %></td>
+        <td><%= operation.getUserName() %></td>
+        <td><%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %></td>
+        <td><%= operation.getExecutionEngine() %>
+        <td><%= operation.getState() %></td>
+        <td><%= operation.getElapsedTime()/1000 %></td>
+        <td><%= operation.getEndTime() == null ? "In Progress" : new Date(operation.getEndTime()) %></td>
+        <% String link = "/query_page?operationId=" + operation.getOperationId(); %>
+        <td>  <a href= <%= link %>>Query Drilldown</a> </td>
     </tr>
-<%
-}
 
+<%
+  }
 %>
 <tr>
-  <td colspan="6">Total number of queries: <%= queries %></td>
+  <td colspan="8">Total number of queries: <%= queries %></td>
 </tr>
 </table>
 </section>
 
-<% 
+<%
  }
 %>
 


Mime
View raw message