hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject hive git commit: HIVE-12550 : Cache and display last N completed queries in HS2 WebUI (Szehon, reviewed by Aihua Xu and Mohit Sabharwal)
Date Fri, 29 Jan 2016 18:37:42 GMT
Repository: hive
Updated Branches:
  refs/heads/master 41fc1874b -> 3db7227fc


HIVE-12550 : Cache and display last N completed queries in HS2 WebUI (Szehon, reviewed by
Aihua Xu and Mohit Sabharwal)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3db7227f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3db7227f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3db7227f

Branch: refs/heads/master
Commit: 3db7227fc7e40ef65d59a8e5d26b9c83bc6fe08c
Parents: 41fc187
Author: Szehon Ho <szehon@cloudera.com>
Authored: Fri Jan 22 15:37:37 2016 -0500
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Fri Jan 29 10:36:34 2016 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  2 +
 pom.xml                                         |  2 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  9 ++++
 .../hive/service/cli/operation/Operation.java   | 12 ++++-
 .../service/cli/operation/OperationManager.java | 49 +++++++++++++++++++-
 .../service/cli/operation/SQLOperation.java     | 22 ++++++++-
 .../service/cli/operation/SQLOperationInfo.java | 48 +++++++++++++++++++
 .../hive-webapps/hiveserver2/hiveserver2.jsp    | 41 +++++++++++++++-
 8 files changed, 180 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bfd88f8..a78f78a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1910,6 +1910,8 @@ public class HiveConf extends Configuration {
         "HTTP/_HOST@EXAMPLE.COM", "The HiveServer2 WebUI SPNEGO service principal.\n" +
         "The special string _HOST will be replaced automatically with \n" +
         "the value of hive.server2.webui.host or the correct host name."),
+    HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES("hive.server2.webui.max.historic.queries", 25,
+        "The maximum number of past queries to show in HiverSever2 WebUI."),
 
     // Tez session settings
     HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2d2a3de..802d3d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,7 +125,7 @@
     <commons-dbcp.version>1.4</commons-dbcp.version>
     <derby.version>10.10.2.0</derby.version>
     <dropwizard.version>3.1.0</dropwizard.version>
-    <guava.version>14.0.1</guava.version>
+    <guava.version>15.0</guava.version>
     <groovy.version>2.4.4</groovy.version>
     <hadoop.version>2.6.0</hadoop.version>
     <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 75187cf..4c89812 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -158,6 +158,9 @@ public class Driver implements CommandProcessor {
   // HS2 operation handle guid string
   private String operationId;
 
+  // For WebUI.  Kept alive after queryPlan is freed.
+  private String savedQueryString;
+
   private boolean checkConcurrency() {
     boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
     if (!supportConcurrency) {
@@ -384,6 +387,7 @@ public class Driver implements CommandProcessor {
     } catch (Exception e) {
       LOG.warn("WARNING! Query command could not be redacted." + e);
     }
+    this.savedQueryString = queryStr;
 
     //holder for parent command type/string when executing reentrant queries
     QueryState queryState = new QueryState();
@@ -1943,6 +1947,11 @@ public class Driver implements CommandProcessor {
     return errorMessage;
   }
 
+
+  public String getQueryString() {
+    return savedQueryString == null ? "Unknown" : savedQueryString;
+  }
+
   /**
    * Set the HS2 operation handle's guid string
    * @param opId base64 encoded guid string

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index d6ac16d..0c263cf 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -75,6 +75,7 @@ public abstract class Operation {
 
   private long operationTimeout;
   private volatile long lastAccessTime;
+  private final long beginTime;
 
   protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
       EnumSet.of(FetchOrientation.FETCH_NEXT,FetchOrientation.FETCH_FIRST);
@@ -92,7 +93,8 @@ public abstract class Operation {
     }
     this.runAsync = runInBackground;
     this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion());
-    lastAccessTime = System.currentTimeMillis();
+    beginTime = System.currentTimeMillis();
+    lastAccessTime = beginTime;
     operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
         HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
     setMetrics(state);
@@ -410,4 +412,12 @@ public abstract class Operation {
        }
     }
   }
+
+  public long getBeginTime() {
+    return beginTime;
+  }
+
+  protected OperationState getState() {
+    return state;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 92135cd..f1ce6f6 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -22,10 +22,14 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.collect.EvictingQueue;
 import org.apache.hadoop.hive.common.metrics.common.Metrics;
 import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
 import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
@@ -61,6 +65,9 @@ public class OperationManager extends AbstractService {
   private final ConcurrentHashMap<OperationHandle, Operation> handleToOperation =
       new ConcurrentHashMap<OperationHandle, Operation>();
 
+  //for displaying historical queries on WebUI
+  private Queue<SQLOperationInfo> historicSqlOperations;
+
   public OperationManager() {
     super(OperationManager.class.getSimpleName());
   }
@@ -73,6 +80,11 @@ public class OperationManager extends AbstractService {
     } else {
       LOG.debug("Operation level logging is turned off");
     }
+    if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
+      hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) >
0) {
+      historicSqlOperations = EvictingQueue.create(
+        hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES));
+    }
     super.init(hiveConf);
   }
 
@@ -174,6 +186,7 @@ public class OperationManager extends AbstractService {
     Operation operation = handleToOperation.get(operationHandle);
     if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
       handleToOperation.remove(operationHandle, operation);
+      cacheOldOperationInfo(operation);
       return operation;
     }
     return null;
@@ -184,7 +197,9 @@ public class OperationManager extends AbstractService {
   }
 
   private Operation removeOperation(OperationHandle opHandle) {
-    return handleToOperation.remove(opHandle);
+    Operation result = handleToOperation.remove(opHandle);
+    cacheOldOperationInfo(result);
+    return result;
   }
 
   public OperationStatus getOperationStatus(OperationHandle opHandle)
@@ -313,4 +328,36 @@ public class OperationManager extends AbstractService {
     }
     return removed;
   }
+
+  //Cache a number of historical operation info, at max number of
+  //hive.server2.webui.max.historic.queries.
+  private void cacheOldOperationInfo(Operation oldOperation) {
+    if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
+      getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES)
> 0) {
+      if (oldOperation instanceof SQLOperation) {
+        SQLOperation query = (SQLOperation) oldOperation;
+        SQLOperationInfo queryInfo = query.getSQLOperationInfo();
+        if (queryInfo != null) {
+          synchronized (historicSqlOperations) {
+            historicSqlOperations.add(queryInfo);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * @return a number of historical SQLOperation info, at max number of
+   * hive.server2.webui.max.historic.queries
+   */
+  public List<SQLOperationInfo> getHistoricalSQLOpInfo() {
+    List<SQLOperationInfo> result = new LinkedList<>();
+    synchronized (historicSqlOperations) {
+      Iterator<SQLOperationInfo> opIterator = historicSqlOperations.iterator();
+      while (opIterator.hasNext()) {
+        result.add(opIterator.next());
+      }
+    }
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index c8a69b9..01b1d3d 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -163,7 +163,7 @@ public class SQLOperation extends ExecuteStatementOperation {
   }
 
   public String getQueryStr() {
-    return driver == null || driver.getPlan() == null ? "Unknown" : driver.getPlan().getQueryStr();
+    return driver == null ? "Unknown" : driver.getQueryString();
   }
 
   private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
@@ -199,6 +199,7 @@ public class SQLOperation extends ExecuteStatementOperation {
   public void runInternal() throws HiveSQLException {
     setState(OperationState.PENDING);
     final HiveConf opConfig = getConfigForOperation();
+
     prepare(opConfig);
     if (!shouldRunAsync()) {
       runQuery(opConfig);
@@ -304,6 +305,7 @@ public class SQLOperation extends ExecuteStatementOperation {
         backgroundHandle.cancel(true);
       }
     }
+
     if (driver != null) {
       driver.close();
       driver.destroy();
@@ -479,4 +481,22 @@ public class SQLOperation extends ExecuteStatementOperation {
     }
     return sqlOperationConf;
   }
+
+  /**
+   * Get summary information of this SQLOperation for display in WebUI.
+   */
+  public SQLOperationInfo getSQLOperationInfo() {
+    try {
+      return new SQLOperationInfo(
+        getParentSession().getUserName(),
+        driver.getQueryString(),
+        getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE),
+        getState(),
+        (int) (System.currentTimeMillis() - getBeginTime()) / 1000,
+        System.currentTimeMillis());
+    } catch (HiveSQLException e) {
+      LOG.warn("Error calcluating SQL Operation Info for webui", e);
+    }
+    return null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
new file mode 100644
index 0000000..179f6dd
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.operation;
+
+import org.apache.hive.service.cli.OperationState;
+
+/**
+ * Used to display some info in the HS2 WebUI.
+ */
+public class SQLOperationInfo {
+  public String userName;
+  public String queryStr;
+  public String executionEngine;
+  public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR)
+  public int elapsedTime;
+  public long endTime;
+
+  public SQLOperationInfo(
+    String userName,
+    String queryStr,
+    String executionEngine,
+    OperationState endState,
+    int elapsedTime,
+    long endTime
+  ) {
+    this.userName = userName;
+    this.queryStr = queryStr;
+    this.executionEngine = executionEngine;
+    this.endState = endState;
+    this.elapsedTime = elapsedTime;
+    this.endTime = endTime;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
----------------------------------------------------------------------
diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
index a91b008..a0b5d2e 100644
--- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
+++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
@@ -24,11 +24,13 @@
   import="org.apache.hive.common.util.HiveVersionInfo"
   import="org.apache.hive.service.cli.operation.Operation"
   import="org.apache.hive.service.cli.operation.SQLOperation"
+  import="org.apache.hive.service.cli.operation.SQLOperationInfo"
   import="org.apache.hive.service.cli.session.SessionManager"
   import="org.apache.hive.service.cli.session.HiveSession"
   import="javax.servlet.ServletContext"
   import="java.util.Collection"
   import="java.util.Date"
+  import="java.util.List"
 %>
 
 <%
@@ -145,7 +147,7 @@ for (Operation operation: operations) {
         <td><%= query.getQueryStr() %></td>
         <td><%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE)
%>
         <td><%= query.getStatus().getState() %></td>
-        <td><%= (currentTime - query.getLastAccessTime())/1000 %></td>
+        <td><%= (currentTime - query.getBeginTime())/1000 %></td>
     </tr>
 <%
   }
@@ -156,6 +158,43 @@ for (Operation operation: operations) {
 </tr>
 </table>
 </section>
+
+
+<section>
+<h2>Last Max <%= conf.get(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES.varname)
%> Completed Queries</h2>
+<table id="attributes_table" class="table table-striped">
+    <tr>
+        <th>User Name</th>
+        <th>Query</th>
+        <th>Execution Engine</th>
+        <th>State</th>
+        <th>Elapsed Time (s)</th>
+        <th>End Time </th>
+    </tr>
+<%
+queries = 0;
+List<SQLOperationInfo> sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo();
+for (SQLOperationInfo sqlOperation: sqlOperations) {
+  queries++;
+%>
+    <tr>
+        <td><%= sqlOperation.userName %></td>
+        <td><%= sqlOperation.queryStr %></td>
+        <td><%= sqlOperation.executionEngine %></td>
+        <td><%= sqlOperation.endState %></td>
+        <td><%= sqlOperation.elapsedTime %></td>
+        <td><%= new Date(sqlOperation.endTime) %></td>
+    </tr>
+<%
+}
+
+%>
+<tr>
+  <td colspan="6">Total number of queries: <%= queries %></td>
+</tr>
+</table>
+</section>
+
 <% 
  }
 %>


Mime
View raw message