Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4790C180C7 for ; Wed, 3 Feb 2016 21:20:31 +0000 (UTC) Received: (qmail 88715 invoked by uid 500); 3 Feb 2016 21:20:28 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 88467 invoked by uid 500); 3 Feb 2016 21:20:28 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 88040 invoked by uid 99); 3 Feb 2016 21:20:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Feb 2016 21:20:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 71694DFB85; Wed, 3 Feb 2016 21:20:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: szehon@apache.org To: commits@hive.apache.org Date: Wed, 03 Feb 2016 21:20:30 -0000 Message-Id: In-Reply-To: <6b0d8d10a1e147019c03a44112d0a816@git.apache.org> References: <6b0d8d10a1e147019c03a44112d0a816@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [4/4] hive git commit: HIVE-12952 : Show query sub-pages on webui (Szehon, reviewed by Aihua Xu and Lenni Kuff) HIVE-12952 : Show query sub-pages on webui (Szehon, reviewed by Aihua Xu and Lenni Kuff) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d94c0b0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d94c0b0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d94c0b0 Branch: refs/heads/master Commit: 2d94c0b0bd9fe88747eb749dba8e1ab51d94a4ff Parents: f2e9edb Author: Szehon Ho Authored: Wed Feb 3 22:18:49 2016 +0100 Committer: Szehon Ho Committed: Wed Feb 3 22:18:49 2016 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 15 + .../apache/hadoop/hive/ql/log/PerfLogger.java | 9 + .../java/org/apache/hive/http/HttpServer.java | 2 +- pom.xml | 12 + .../java/org/apache/hadoop/hive/ql/Driver.java | 48 +- .../org/apache/hadoop/hive/ql/QueryDisplay.java | 253 +++ .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- service/pom.xml | 51 +- .../org/apache/hive/tmpl/QueryProfileTmpl.jamon | 298 +++ .../hive/service/cli/operation/Operation.java | 4 + .../service/cli/operation/OperationManager.java | 82 +- .../service/cli/operation/SQLOperation.java | 39 +- .../cli/operation/SQLOperationDisplay.java | 99 + .../cli/operation/SQLOperationDisplayCache.java | 39 + .../service/cli/operation/SQLOperationInfo.java | 48 - .../apache/hive/service/server/HiveServer2.java | 2 + .../service/servlet/QueryProfileServlet.java | 49 + .../hive-webapps/hiveserver2/hiveserver2.jsp | 72 +- .../hive-webapps/static/js/bootstrap.js | 1999 ++++++++++++++++++ .../hive-webapps/static/js/bootstrap.min.js | 6 + .../hive-webapps/static/js/jquery.min.js | 2 + .../src/resources/hive-webapps/static/js/tab.js | 38 + 22 files changed, 3028 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6678de6..73e6c21 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3741,6 +3741,21 @@ public class HiveConf extends Configuration { } } + /** + * @return true if HS2 webui is enabled + */ + public boolean isWebUiEnabled() { + return this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0; + } + + /** + * @return true if HS2 webui query-info cache is enabled + */ + public boolean isWebUiQueryInfoCacheEnabled() { + return isWebUiEnabled() && this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0; + } + + public static boolean isLoadMetastoreConfig() { return loadMetastoreConfig; } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index d4194cf..8fa5cbf 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.log; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.common.metrics.common.Metrics; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -222,4 +223,12 @@ public class PerfLogger { LOG.warn("Error recording metrics", e); } } + + public ImmutableMap getStartTimes() { + return ImmutableMap.copyOf(startTimes); + } + + public ImmutableMap getEndTimes() { + return ImmutableMap.copyOf(endTimes); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/common/src/java/org/apache/hive/http/HttpServer.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 9e23b11..b8836de 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -435,7 +435,7 @@ public class HttpServer { * @param pathSpec The path spec for the servlet * @param clazz The servlet class */ - void addServlet(String name, String pathSpec, + public void addServlet(String name, String pathSpec, Class clazz) { ServletHolder holder = new ServletHolder(clazz); if (name != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8c2257f..3c06188 100644 --- a/pom.xml +++ b/pom.xml @@ -140,6 +140,8 @@ 2.4.2 5.5.23 + 2.3.4 + 2.3.1 0.3.2 3.0.0.v201112011016 5.5.1 @@ -725,6 +727,11 @@ + + org.jamon + jamon-runtime + ${jamon-runtime.version} + @@ -1064,6 +1071,11 @@ + + org.jamon + jamon-maven-plugin + ${jamon.plugin.version} + http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index bcf62a7..7147a9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -53,7 +54,6 @@ import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -159,7 +159,7 @@ public class Driver implements CommandProcessor { private String operationId; // For WebUI. Kept alive after queryPlan is freed. - private String savedQueryString; + private final QueryDisplay queryDisplay = new QueryDisplay(); private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); @@ -387,7 +387,6 @@ public class Driver implements CommandProcessor { } catch (Exception e) { LOG.warn("WARNING! Query command could not be redacted." + e); } - this.savedQueryString = queryStr; //holder for parent command type/string when executing reentrant queries QueryState queryState = new QueryState(); @@ -409,6 +408,10 @@ public class Driver implements CommandProcessor { conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId); } + //save some info for webUI for use after plan is freed + this.queryDisplay.setQueryStr(queryStr); + this.queryDisplay.setQueryId(queryId); + LOG.info("Compiling command(queryId=" + queryId + "): " + queryStr); SessionState.get().setupQueryCurrentTimestamp(); @@ -519,11 +522,17 @@ public class Driver implements CommandProcessor { } } - if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT) || + conf.isWebUiQueryInfoCacheEnabled()) { String explainOutput = getExplainOutput(sem, plan, tree); if (explainOutput != null) { - LOG.info("EXPLAIN output for queryid " + queryId + " : " + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + LOG.info("EXPLAIN output for queryid " + queryId + " : " + explainOutput); + } + if (conf.isWebUiQueryInfoCacheEnabled()) { + queryDisplay.setExplainPlan(explainOutput); + } } } return 0; @@ -553,18 +562,20 @@ public class Driver implements CommandProcessor { // since it exceeds valid range of shell return values } finally { double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.COMPILE)/1000.00; - dumpMetaCallTimingWithoutEx("compilation"); + ImmutableMap compileHMSTimings = dumpMetaCallTimingWithoutEx("compilation"); + queryDisplay.setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings); restoreSession(queryState); LOG.info("Completed compiling command(queryId=" + queryId + "); Time taken: " + duration + " seconds"); } } - private void dumpMetaCallTimingWithoutEx(String phase) { + private ImmutableMap dumpMetaCallTimingWithoutEx(String phase) { try { - Hive.get().dumpAndClearMetaCallTiming(phase); + return Hive.get().dumpAndClearMetaCallTiming(phase); } catch (HiveException he) { LOG.warn("Caught exception attempting to write metadata call information " + he, he); } + return null; } /** @@ -1186,6 +1197,13 @@ public class Driver implements CommandProcessor { + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } + + //Save compile-time PerfLogging for WebUI. + //Execution-time Perf logs are done by either another thread's PerfLogger + //or a reset PerfLogger. + PerfLogger perfLogger = SessionState.getPerfLogger(); + queryDisplay.setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes()); + queryDisplay.setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes()); return ret; } @@ -1343,6 +1361,8 @@ public class Driver implements CommandProcessor { } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN); + queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes()); + queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes()); // Take all the driver run hooks and post-execute them. try { @@ -1414,6 +1434,7 @@ public class Driver implements CommandProcessor { } private CommandProcessorResponse createProcessorResponse(int ret) { + queryDisplay.setErrorMessage(errorMessage); return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError); } @@ -1544,6 +1565,7 @@ public class Driver implements CommandProcessor { // Launch upto maxthreads tasks Task task; while ((task = driverCxt.getRunnable(maxthreads)) != null) { + queryDisplay.addTask(task); TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, driverCxt); if (!runner.isRunning()) { break; @@ -1556,6 +1578,7 @@ public class Driver implements CommandProcessor { continue; } hookContext.addCompleteTask(tskRun); + queryDisplay.setTaskCompleted(tskRun.getTask().getId(), tskRun.getTaskResult()); Task tsk = tskRun.getTask(); TaskResult result = tskRun.getTaskResult(); @@ -1694,9 +1717,11 @@ public class Driver implements CommandProcessor { if (noName) { conf.set(MRJobConfig.JOB_NAME, ""); } - dumpMetaCallTimingWithoutEx("execution"); double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE)/1000.00; + ImmutableMap executionHMSTimings = dumpMetaCallTimingWithoutEx("execution"); + queryDisplay.setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings); + Map stats = SessionState.get().getMapRedStats(); if (stats != null && !stats.isEmpty()) { long totalCpu = 0; @@ -1952,8 +1977,8 @@ public class Driver implements CommandProcessor { } - public String getQueryString() { - return savedQueryString == null ? "Unknown" : savedQueryString; + public QueryDisplay getQueryDisplay() { + return queryDisplay; } /** @@ -1963,5 +1988,4 @@ public class Driver implements CommandProcessor { public void setOperationId(String opId) { this.operationId = opId; } - } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java new file mode 100644 index 0000000..c87c825 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskResult; +import org.apache.hadoop.hive.ql.plan.api.StageType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Some limited query information to save for WebUI. + * + * The class is synchronized, as WebUI may access information about a running query. + */ +public class QueryDisplay { + + // Member variables + private String queryStr; + private String explainPlan; + private String errorMessage; + private String queryId; + + private final Map> hmsTimingMap = new HashMap(); + private final Map> perfLogStartMap = new HashMap(); + private final Map> perfLogEndMap = new HashMap(); + + private final LinkedHashMap tasks = new LinkedHashMap(); + + //Inner classes + public static enum Phase { + COMPILATION, + EXECUTION, + } + + public static class TaskInfo { + private Integer returnVal; //if set, determines that task is complete. + private String errorMsg; + private long endTime; + + final long beginTime; + final String taskId; + final StageType taskType; + final String name; + final boolean requireLock; + final boolean retryIfFail; + + public TaskInfo (Task task) { + beginTime = System.currentTimeMillis(); + taskId = task.getId(); + taskType = task.getType(); + name = task.getName(); + requireLock = task.requireLock(); + retryIfFail = task.ifRetryCmdWhenFail(); + } + + public synchronized String getStatus() { + if (returnVal == null) { + return "Running"; + } else if (returnVal == 0) { + return "Success, ReturnVal 0"; + } else { + return "Failure, ReturnVal " + String.valueOf(returnVal); + } + } + + public synchronized long getElapsedTime() { + if (endTime == 0) { + return System.currentTimeMillis() - beginTime; + } else { + return endTime - beginTime; + } + } + + public synchronized String getErrorMsg() { + return errorMsg; + } + + public synchronized long getEndTime() { + return endTime; + } + + //Following methods do not need to be synchronized, because they are final fields. + public long getBeginTime() { + return beginTime; + } + + public String getTaskId() { + return taskId; + } + + public StageType getTaskType() { + return taskType; + } + + public String getName() { + return name; + } + + public boolean isRequireLock() { + return requireLock; + } + + public boolean isRetryIfFail() { + return retryIfFail; + } + } + + public synchronized void addTask(Task task) { + tasks.put(task.getId(), new TaskInfo(task)); + } + + public synchronized void setTaskCompleted(String taskId, TaskResult result) { + TaskInfo taskInfo = tasks.get(taskId); + if (taskInfo != null) { + taskInfo.returnVal = result.getExitVal(); + if (result.getTaskError() != null) { + taskInfo.errorMsg = result.getTaskError().toString(); + } + taskInfo.endTime = System.currentTimeMillis(); + } + } + + public synchronized List getTaskInfos() { + List taskInfos = new ArrayList(); + taskInfos.addAll(tasks.values()); + return taskInfos; + } + + public synchronized void setQueryStr(String queryStr) { + this.queryStr = queryStr; + } + + public synchronized String getQueryString() { + return returnStringOrUnknown(queryStr); + } + + public synchronized String getExplainPlan() { + return returnStringOrUnknown(explainPlan); + } + + public synchronized void setExplainPlan(String explainPlan) { + this.explainPlan = explainPlan; + } + + /** + * @param phase phase of query + * @return map of HMS Client method-calls and duration in miliseconds, during given phase. + */ + public synchronized Map getHmsTimings(Phase phase) { + return hmsTimingMap.get(phase); + } + + /** + * @param phase phase of query + * @param hmsTimings map of HMS Client method-calls and duration in miliseconds, during given phase. + */ + public synchronized void setHmsTimings(Phase phase, ImmutableMap hmsTimings) { + hmsTimingMap.put(phase, hmsTimings); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and start time in miliseconds, during given phase. + */ + public synchronized Map getPerfLogStarts(Phase phase) { + return perfLogStartMap.get(phase); + } + + /** + * @param phase phase of query + * @param perfLogStarts map of PerfLogger call-trace name and start time in miliseconds, during given phase. + */ + public synchronized void setPerfLogStarts(Phase phase, ImmutableMap perfLogStarts) { + perfLogStartMap.put(phase, perfLogStarts); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and end time in miliseconds, during given phase. + */ + public synchronized Map getPerfLogEnds(Phase phase) { + return perfLogEndMap.get(phase); + } + + /** + * @param phase phase of query + * @param perfLogEnds map of PerfLogger call-trace name and end time in miliseconds, during given phase. + */ + public synchronized void setPerfLogEnds(Phase phase, ImmutableMap perfLogEnds) { + perfLogEndMap.put(phase, perfLogEnds); + } + + /** + * @param phase phase of query + * @return map of PerfLogger call-trace name and duration in miliseconds, during given phase. + */ + public synchronized Map getPerfLogTimes(Phase phase) { + Map times = new HashMap<>(); + Map startTimes = perfLogStartMap.get(phase); + Map endTimes = perfLogEndMap.get(phase); + if (endTimes != null && startTimes != null) { + for (String timeKey : endTimes.keySet()) { + Long endTime = endTimes.get(timeKey); + Long startTime = startTimes.get(timeKey); + if (startTime != null) { + times.put(timeKey, endTime - startTime); + } + } + } + return times; + } + + public synchronized String getErrorMessage() { + return errorMessage; + } + + public synchronized void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public synchronized String getQueryId() { + return returnStringOrUnknown(queryId); + } + + public synchronized void setQueryId(String queryId) { + this.queryId = queryId; + } + + private String returnStringOrUnknown(String s) { + return s == null ? "UNKNOWN" : s; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0bab769..4e574f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -44,6 +44,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -3400,7 +3401,7 @@ private void constructOneLBLocationMap(FileStatus fSta, metaCallTimeMap.clear(); } - public void dumpAndClearMetaCallTiming(String phase) { + public ImmutableMap dumpAndClearMetaCallTiming(String phase) { boolean phaseInfoLogged = false; if (LOG.isDebugEnabled()) { phaseInfoLogged = logDumpPhase(phase); @@ -3420,7 +3421,10 @@ private void constructOneLBLocationMap(FileStatus fSta, } } } + + ImmutableMap result = ImmutableMap.copyOf(metaCallTimeMap); metaCallTimeMap.clear(); + return result; } private boolean logDumpPhase(String phase) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index b2e3a84..e3f61d0 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -128,7 +128,13 @@ ${hadoop.version} true - + + org.jamon + jamon-runtime + ${jamon-runtime.version} + + + org.apache.hive hive-exec @@ -219,7 +225,48 @@ - + + org.jamon + jamon-maven-plugin + + + generate-sources + + translate + + + src/jamon + target/generated-jamon + + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + + + org.jamon.project.jamonnature + + + org.jamon.project.templateBuilder + org.eclipse.jdt.core.javabuilder + org.jamon.project.markerUpdater + + + + .settings/org.jamon.prefs + # now + eclipse.preferences.version=1 + templateSourceDir=src/main/jamon + templateOutputDir=target/generated-jamon + + + + + + org.apache.maven.plugins maven-jar-plugin http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon ---------------------------------------------------------------------- diff --git a/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon new file mode 100644 index 0000000..c513689 --- /dev/null +++ b/service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon @@ -0,0 +1,298 @@ +<%doc> + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%args> +SQLOperationDisplay sod; + +<%import> +java.util.*; +org.apache.hadoop.hive.ql.QueryDisplay; +org.apache.hive.service.cli.operation.SQLOperationDisplay; + + + + + + + HiveServer2 + + + + + + + + + +
+
+
+ + Hive Logo +
+
+ +
+
+
+ + + + <%if sod == null %> +
+

Query not found. It may have been deleted, increase hive.server2.webui.max.historic.queries + to retain more historic query information.

+
+ <%else> + + +
+
+
+

Query Information: <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %>

+
+
+
+ + +
+ +
+
+ <& baseProfile; sod = sod &> +
+
+ <& stages; sod = sod &> +
+
+ <& queryPlan; sod = sod &> +
+
+ <& perfLogging; sod = sod &> +
+
+
+ + + +
+
+ + + + + + +<%def baseProfile> +<%args> + SQLOperationDisplay sod; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getErrorMessage() != null %> + + + + + +
User Name<% sod.getUserName() %>
Query String<% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryString() %>
Query Id<% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getQueryId() %>
Execution Engine<% sod.getExecutionEngine() %> +
State<% sod.getState() %>
Begin Time<% new Date(sod.getBeginTime()) %>
Elapsed Time (s)<% sod.getElapsedTime()/1000 %>
End Time<% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %>
Error<% sod.getEndTime() == null ? "In Progress" : new Date(sod.getEndTime()) %>
+ + +<%def stages> +<%args> + SQLOperationDisplay sod; + + + + + + + + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getTaskInfos() != null %> + <%for QueryDisplay.TaskInfo taskInfo : sod.getQueryDisplay().getTaskInfos() %> + + + + + + + + + + +
Stage IdStatusBegin TimeEnd TimeElapsed Time (s)Requires LockRetry If Fail
<% taskInfo.getTaskId() + ":" + taskInfo.getTaskType() %><% taskInfo.getStatus() %><% new Date(taskInfo.getBeginTime()) %> + <% taskInfo.getEndTime() == 0 ? "" : new Date(taskInfo.getEndTime()) %><% taskInfo.getElapsedTime()/1000 %> (s) <% taskInfo.isRequireLock() %><% taskInfo.isRetryIfFail() %>
+ + + +<%def queryPlan> +<%args> + SQLOperationDisplay sod; + +
+
Explain plan
+
+ 
+        <% sod.getQueryDisplay() == null ? "Unknown" : sod.getQueryDisplay().getExplainPlan() %>
+
+
+
+ + + +<%def perfLogging> +<%args> + SQLOperationDisplay sod; + +
+

Compile-time metadata operations

+ + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION) != null %> + <%for Map.Entry time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.COMPILATION).entrySet() %> + + + + + + +
Call NameTime (ms)
<% time.getKey() %><% time.getValue() %>
+
+ +
+

Execution-time metadata operations

+ + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION) != null %> + <%for Map.Entry time : sod.getQueryDisplay().getHmsTimings(QueryDisplay.Phase.EXECUTION).entrySet() %> + + + + + + +
Call NameTime (ms)
<% time.getKey() %><% time.getValue() %>
+
+ +
+

Compile-Time Perf-Logger

+ + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION) != null %> + <%for Map.Entry time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.COMPILATION).entrySet() %> + + + + + + +
Compile-time Call NameTime (ms)
<% time.getKey() %><% time.getValue() %>
+
+ +
+

Execution-Time Perf-Logger

+ + + + + + + <%if sod.getQueryDisplay() != null && sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION) != null %> + <%for Map.Entry time : sod.getQueryDisplay().getPerfLogTimes(QueryDisplay.Phase.EXECUTION).entrySet() %> + + + + + + +
Execution-time Call NameTime (ms)
<% time.getKey() %><% time.getValue() %>
+
+ + + + + + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/Operation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 0c263cf..8340202 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -157,6 +157,7 @@ public abstract class Operation { state.validateTransition(newState); this.state = newState; setMetrics(state); + onNewState(state); this.lastAccessTime = System.currentTimeMillis(); return this.state; } @@ -420,4 +421,7 @@ public abstract class Operation { protected OperationState getState() { return state; } + + protected void onNewState(OperationState state) { + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index f1ce6f6..38f27ef 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -22,7 +22,9 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -65,8 +67,10 @@ public class OperationManager extends AbstractService { private final ConcurrentHashMap handleToOperation = new ConcurrentHashMap(); - //for displaying historical queries on WebUI - private Queue historicSqlOperations; + //Following fields for displaying queries on WebUI + private Object webuiLock = new Object(); + private SQLOperationDisplayCache historicSqlOperations; + private Map liveSqlOperations = new LinkedHashMap(); public OperationManager() { super(OperationManager.class.getSimpleName()); @@ -80,9 +84,8 @@ public class OperationManager extends AbstractService { } else { LOG.debug("Operation level logging is turned off"); } - if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && - hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { - historicSqlOperations = EvictingQueue.create( + if (hiveConf.isWebUiQueryInfoCacheEnabled()) { + historicSqlOperations = new SQLOperationDisplayCache( hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES)); } super.init(hiveConf); @@ -186,7 +189,11 @@ public class OperationManager extends AbstractService { Operation operation = handleToOperation.get(operationHandle); if (operation != null && operation.isTimedOut(System.currentTimeMillis())) { handleToOperation.remove(operationHandle, operation); - cacheOldOperationInfo(operation); + synchronized (webuiLock) { + String opKey = operationHandle.getHandleIdentifier().toString(); + SQLOperationDisplay display = liveSqlOperations.remove(opKey); + historicSqlOperations.put(opKey, display); + } return operation; } return null; @@ -194,11 +201,21 @@ public class OperationManager extends AbstractService { private void addOperation(Operation operation) { handleToOperation.put(operation.getHandle(), operation); + if (operation instanceof SQLOperation) { + synchronized (webuiLock) { + liveSqlOperations.put(operation.getHandle().getHandleIdentifier().toString(), + ((SQLOperation) operation).getSQLOperationDisplay()); + } + } } private Operation removeOperation(OperationHandle opHandle) { Operation result = handleToOperation.remove(opHandle); - cacheOldOperationInfo(result); + synchronized (webuiLock) { + String opKey = opHandle.getHandleIdentifier().toString(); + SQLOperationDisplay display = liveSqlOperations.remove(opKey); + historicSqlOperations.put(opKey, display); + } return result; } @@ -329,35 +346,40 @@ public class OperationManager extends AbstractService { return removed; } - //Cache a number of historical operation info, at max number of - //hive.server2.webui.max.historic.queries. - private void cacheOldOperationInfo(Operation oldOperation) { - if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && - getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { - if (oldOperation instanceof SQLOperation) { - SQLOperation query = (SQLOperation) oldOperation; - SQLOperationInfo queryInfo = query.getSQLOperationInfo(); - if (queryInfo != null) { - synchronized (historicSqlOperations) { - historicSqlOperations.add(queryInfo); - } - } - } + /** + * @return displays representing a number of historical SQLOperations, at max number of + * hive.server2.webui.max.historic.queries + */ + public List getHistoricalSQLOperations() { + List result = new LinkedList<>(); + synchronized (webuiLock) { + result.addAll(historicSqlOperations.values()); } + return result; } /** - * @return a number of historical SQLOperation info, at max number of - * hive.server2.webui.max.historic.queries + * @return displays representing live SQLOperations */ - public List getHistoricalSQLOpInfo() { - List result = new LinkedList<>(); - synchronized (historicSqlOperations) { - Iterator opIterator = historicSqlOperations.iterator(); - while (opIterator.hasNext()) { - result.add(opIterator.next()); - } + public List getLiveSqlOperations() { + List result = new LinkedList<>(); + synchronized (webuiLock) { + result.addAll(liveSqlOperations.values()); } return result; } + + /** + * @param handle handle of SQLOperation. + * @return display representing a particular SQLOperation. + */ + public SQLOperationDisplay getSQLOperationDisplay(String handle) { + synchronized (webuiLock) { + SQLOperationDisplay result = liveSqlOperations.get(handle); + if (result != null) { + return result; + } + return historicSqlOperations.get(handle); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 01b1d3d..3fbbb70 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -34,8 +34,6 @@ import java.util.concurrent.RejectedExecutionException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.CharEncoding; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.CommandNeedRetryException; @@ -79,11 +77,20 @@ public class SQLOperation extends ExecuteStatementOperation { private SerDe serde = null; private boolean fetchStarted = false; + //Display for WebUI. + private SQLOperationDisplay sqlOpDisplay; + + public SQLOperation(HiveSession parentSession, String statement, Map confOverlay, boolean runInBackground) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); setupSessionIO(parentSession.getSessionState()); + try { + sqlOpDisplay = new SQLOperationDisplay(this); + } catch (HiveSQLException e) { + LOG.warn("Error calcluating SQL Operation Display for webui", e); + } } private void setupSessionIO(SessionState sessionState) { @@ -111,6 +118,7 @@ public class SQLOperation extends ExecuteStatementOperation { try { driver = new Driver(sqlOperationConf, getParentSession().getUserName()); + sqlOpDisplay.setQueryDisplay(driver.getQueryDisplay()); // set the operation handle information in Driver, so that thrift API users // can use the operation handle they receive, to lookup query information in @@ -162,10 +170,6 @@ public class SQLOperation extends ExecuteStatementOperation { } } - public String getQueryStr() { - return driver == null ? "Unknown" : driver.getQueryString(); - } - private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException { try { // In Hive server mode, we are not able to retry in the FetchTask @@ -485,18 +489,17 @@ public class SQLOperation extends ExecuteStatementOperation { /** * Get summary information of this SQLOperation for display in WebUI. */ - public SQLOperationInfo getSQLOperationInfo() { - try { - return new SQLOperationInfo( - getParentSession().getUserName(), - driver.getQueryString(), - getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE), - getState(), - (int) (System.currentTimeMillis() - getBeginTime()) / 1000, - System.currentTimeMillis()); - } catch (HiveSQLException e) { - LOG.warn("Error calcluating SQL Operation Info for webui", e); + public SQLOperationDisplay getSQLOperationDisplay() { + return sqlOpDisplay; + } + + @Override + protected void onNewState(OperationState state) { + if (state == OperationState.CLOSED) { + sqlOpDisplay.closed(); + } else { + //CLOSED state not interesting, state before (FINISHED, ERROR) is. + sqlOpDisplay.updateState(state); } - return null; } } http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java new file mode 100644 index 0000000..d2ca1e7 --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplay.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryDisplay; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationState; + +/** + * Used to display some info in the HS2 WebUI. + * + * The class is synchronized, as WebUI may access information about a running query. + */ +public class SQLOperationDisplay { + public final String userName; + public final String executionEngine; + public final long beginTime; + public final String operationId; + + public Long endTime; + public OperationState state; + public QueryDisplay queryDisplay; + + public SQLOperationDisplay(SQLOperation sqlOperation) throws HiveSQLException { + this.state = sqlOperation.getState(); + this.userName = sqlOperation.getParentSession().getUserName(); + this.executionEngine = sqlOperation.getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE); + this.beginTime = System.currentTimeMillis(); + this.operationId = sqlOperation.getHandle().getHandleIdentifier().toString(); + } + + public synchronized long getElapsedTime() { + if (isRunning()) { + return System.currentTimeMillis() - beginTime; + } else { + return endTime - beginTime; + } + } + + public synchronized boolean isRunning() { + return endTime == null; + } + + public synchronized QueryDisplay getQueryDisplay() { + return queryDisplay; + } + + public synchronized void setQueryDisplay(QueryDisplay queryDisplay) { + this.queryDisplay = queryDisplay; + } + + public String getUserName() { + return userName; + } + + public String getExecutionEngine() { + return executionEngine; + } + + public synchronized OperationState getState() { + return state; + } + + public long getBeginTime() { + return beginTime; + } + + public synchronized Long getEndTime() { + return endTime; + } + + public synchronized void updateState(OperationState state) { + this.state = state; + } + + public String getOperationId() { + return operationId; + } + + public synchronized void closed() { + this.endTime = System.currentTimeMillis(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java new file mode 100644 index 0000000..4a33d37 --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationDisplayCache.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Cache some SQLOperation information for WebUI + */ +public class SQLOperationDisplayCache extends LinkedHashMap { + + private final int capacity; + + public SQLOperationDisplayCache(int capacity) { + super(capacity + 1, 1.1f, false); + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > capacity; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java deleted file mode 100644 index 179f6dd..0000000 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.service.cli.operation; - -import org.apache.hive.service.cli.OperationState; - -/** - * Used to display some info in the HS2 WebUI. - */ -public class SQLOperationInfo { - public String userName; - public String queryStr; - public String executionEngine; - public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR) - public int elapsedTime; - public long endTime; - - public SQLOperationInfo( - String userName, - String queryStr, - String executionEngine, - OperationState endState, - int elapsedTime, - long endTime - ) { - this.userName = userName; - this.queryStr = queryStr; - this.executionEngine = executionEngine; - this.endState = endState; - this.elapsedTime = elapsedTime; - this.endTime = endTime; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/server/HiveServer2.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 958458f..c601614 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -64,6 +64,7 @@ import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; +import org.apache.hive.service.servlet.QueryProfileServlet; import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -180,6 +181,7 @@ public class HiveServer2 extends CompositeService { builder.setUseSPNEGO(true); } webServer = builder.build(); + webServer.addServlet("query_page", "/query_page", QueryProfileServlet.class); } } } catch (IOException ie) { http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java new file mode 100644 index 0000000..74a374d --- /dev/null +++ b/service/src/java/org/apache/hive/service/servlet/QueryProfileServlet.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.servlet; + +import org.apache.hive.service.cli.operation.OperationManager; +import org.apache.hive.service.cli.operation.SQLOperationDisplay; +import org.apache.hive.service.cli.session.SessionManager; +import org.apache.hive.tmpl.QueryProfileTmpl; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Renders a query page + */ +public class QueryProfileServlet extends HttpServlet { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String opId = (String) request.getParameter("operationId"); + ServletContext ctx = getServletContext(); + SessionManager sessionManager = + (SessionManager)ctx.getAttribute("hive.sm"); + OperationManager opManager = sessionManager.getOperationManager(); + SQLOperationDisplay sod = opManager.getSQLOperationDisplay(opId); + + new QueryProfileTmpl().render(response.getWriter(), sod); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/2d94c0b0/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp ---------------------------------------------------------------------- diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp index a0b5d2e..8b46550 100644 --- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp +++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp @@ -24,7 +24,7 @@ import="org.apache.hive.common.util.HiveVersionInfo" import="org.apache.hive.service.cli.operation.Operation" import="org.apache.hive.service.cli.operation.SQLOperation" - import="org.apache.hive.service.cli.operation.SQLOperationInfo" + import="org.apache.hive.service.cli.operation.SQLOperationDisplay" import="org.apache.hive.service.cli.session.SessionManager" import="org.apache.hive.service.cli.session.HiveSession" import="javax.servlet.ServletContext" @@ -132,29 +132,32 @@ for (HiveSession hiveSession: hiveSessions) { Query Execution Engine State + Begin Time Elapsed Time (s) + Drilldown Link -<% -int queries = 0; -Collection operations = sessionManager.getOperations(); -for (Operation operation: operations) { - if (operation instanceof SQLOperation) { - SQLOperation query = (SQLOperation) operation; - queries++; -%> + <% + int queries = 0; + Collection operations = sessionManager.getOperationManager().getLiveSqlOperations(); + for (SQLOperationDisplay operation : operations) { + queries++; + %> - <%= query.getParentSession().getUserName() %> - <%= query.getQueryStr() %> - <%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %> - <%= query.getStatus().getState() %> - <%= (currentTime - query.getBeginTime())/1000 %> + <%= operation.getUserName() %> + <%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %> + <%= operation.getExecutionEngine() %> + <%= operation.getState() %> + <%= new Date(operation.getBeginTime()) %> + <%= operation.getElapsedTime()/1000 %> + <% String link = "/query_page?operationId=" + operation.getOperationId(); %> + >Query Drilldown + <% } -} %> - Total number of queries: <%= queries %> + Total number of queries: <%= queries %> @@ -169,33 +172,36 @@ for (Operation operation: operations) { Execution Engine State Elapsed Time (s) - End Time + End Time + Drilldown Link -<% -queries = 0; -List sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo(); -for (SQLOperationInfo sqlOperation: sqlOperations) { - queries++; -%> + <% + queries = 0; + operations = sessionManager.getOperationManager().getHistoricalSQLOperations(); + for (SQLOperationDisplay operation : operations) { + queries++; + %> - <%= sqlOperation.userName %> - <%= sqlOperation.queryStr %> - <%= sqlOperation.executionEngine %> - <%= sqlOperation.endState %> - <%= sqlOperation.elapsedTime %> - <%= new Date(sqlOperation.endTime) %> + <%= operation.getUserName() %> + <%= operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString() %> + <%= operation.getExecutionEngine() %> + <%= operation.getState() %> + <%= operation.getElapsedTime()/1000 %> + <%= operation.getEndTime() == null ? "In Progress" : new Date(operation.getEndTime()) %> + <% String link = "/query_page?operationId=" + operation.getOperationId(); %> + >Query Drilldown -<% -} +<% + } %> - Total number of queries: <%= queries %> + Total number of queries: <%= queries %> -<% +<% } %>