Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BFD1D10168 for ; Tue, 8 Apr 2014 08:01:21 +0000 (UTC) Received: (qmail 51637 invoked by uid 500); 8 Apr 2014 08:01:21 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 51581 invoked by uid 500); 8 Apr 2014 08:01:18 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 51569 invoked by uid 99); 8 Apr 2014 08:01:18 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Apr 2014 08:01:18 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A0BEE94DBF2; Tue, 8 Apr 2014 08:01:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hyunsik@apache.org To: commits@tajo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: git commit: TAJO-738: NPE occurs when failed in QueryMaster's GlobalPlanner.build(). (hyoungjunkim via hyunsik) Date: Tue, 8 Apr 2014 08:01:17 +0000 (UTC) Repository: tajo Updated Branches: refs/heads/branch-0.8.0 2c1064ce6 -> 6e50b57c4 TAJO-738: NPE occurs when failed in QueryMaster's GlobalPlanner.build(). (hyoungjunkim via hyunsik) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6e50b57c Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6e50b57c Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6e50b57c Branch: refs/heads/branch-0.8.0 Commit: 6e50b57c4f8feac8ec72fad0dc39fe3886461f64 Parents: 2c1064c Author: Hyunsik Choi Authored: Tue Apr 8 16:36:07 2014 +0900 Committer: Hyunsik Choi Committed: Tue Apr 8 16:54:20 2014 +0900 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../main/java/org/apache/tajo/cli/TajoCli.java | 3 + .../tajo/master/querymaster/QueryMaster.java | 36 +++++--- .../master/querymaster/QueryMasterTask.java | 95 +++++++++++--------- .../tajo/worker/TajoWorkerClientService.java | 24 +++-- .../src/main/resources/webapps/admin/query.jsp | 6 +- .../src/main/resources/webapps/worker/index.jsp | 17 ++-- .../resources/webapps/worker/querydetail.jsp | 20 ++++- 8 files changed, 130 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5e91b09..03ec046 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -303,6 +303,9 @@ Release 0.8.0 - unreleased BUG FIXES + TAJO-738: NPE occur when failed in QueryMaster's GlobalPlanner.build(). + (hyoungjunkim via hyunsik) + TAJO-739: A subquery with the same column alias caused planning error. (hyoungjunkim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index 426c115..2a49d0b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -366,6 +366,9 @@ public class TajoCli { if (status.getState() == QueryState.QUERY_ERROR) { sout.println("Internal error!"); + if(status.getErrorMessage() != null && !status.getErrorMessage().isEmpty()) { + sout.println(status.getErrorMessage()); + } } else if (status.getState() == QueryState.QUERY_FAILED) { sout.println("Query failed!"); } else if (status.getState() == QueryState.QUERY_KILLED) { http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java index 6efc358..fc0c5b3 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java @@ -355,17 +355,21 @@ public class QueryMaster extends CompositeService implements EventHandler { } private TajoHeartbeat buildTajoHeartBeat(QueryMasterTask queryMasterTask) { - TajoHeartbeat queryHeartbeat = TajoHeartbeat.newBuilder() - .setTajoWorkerHost(workerContext.getQueryMasterManagerService().getBindAddr().getHostName()) - .setTajoQueryMasterPort(workerContext.getQueryMasterManagerService().getBindAddr().getPort()) - .setPeerRpcPort(workerContext.getPeerRpcPort()) - .setTajoWorkerClientPort(workerContext.getTajoWorkerClientService().getBindAddr().getPort()) - .setState(queryMasterTask.getState()) - .setQueryId(queryMasterTask.getQueryId().getProto()) - .setQueryProgress(queryMasterTask.getQuery().getProgress()) - .setQueryFinishTime(queryMasterTask.getQuery().getFinishTime()) - .build(); - return queryHeartbeat; + TajoHeartbeat.Builder builder = TajoHeartbeat.newBuilder(); + + builder.setTajoWorkerHost(workerContext.getQueryMasterManagerService().getBindAddr().getHostName()); + builder.setTajoQueryMasterPort(workerContext.getQueryMasterManagerService().getBindAddr().getPort()); + builder.setPeerRpcPort(workerContext.getPeerRpcPort()); + builder.setTajoWorkerClientPort(workerContext.getTajoWorkerClientService().getBindAddr().getPort()); + builder.setState(queryMasterTask.getState()); + builder.setQueryId(queryMasterTask.getQueryId().getProto()); + + if (queryMasterTask.getQuery() != null) { + builder.setQueryProgress(queryMasterTask.getQuery().getProgress()); + builder.setQueryFinishTime(queryMasterTask.getQuery().getFinishTime()); + } + + return builder.build(); } private class QueryStartEventHandler implements EventHandler { @@ -376,10 +380,18 @@ public class QueryMaster extends CompositeService implements EventHandler { event.getQueryId(), event.getSession(), event.getQueryContext(), event.getSql(), event.getLogicalPlanJson()); queryMasterTask.init(systemConf); - queryMasterTask.start(); + if (!queryMasterTask.isInitError()) { + queryMasterTask.start(); + } + synchronized(queryMasterTasks) { queryMasterTasks.put(event.getQueryId(), queryMasterTask); } + + if (queryMasterTask.isInitError()) { + queryMasterContext.stopQuery(queryMasterTask.getQueryId()); + return; + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 79b4a08..271eaf9 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.util.Clock; import org.apache.tajo.*; @@ -109,6 +110,8 @@ public class QueryMasterTask extends CompositeService { private TajoMetrics queryMetrics; + private Throwable initError; + public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, QueryId queryId, Session session, QueryContext queryContext, String sql, String logicalPlanJson) { @@ -153,8 +156,9 @@ public class QueryMasterTask extends CompositeService { queryMetrics = new TajoMetrics(queryId.toString()); super.init(systemConf); - } catch (IOException e) { - LOG.error(e.getMessage(), e); + } catch (Throwable t) { + LOG.error(t.getMessage(), t); + initError = t; } } @@ -294,49 +298,42 @@ public class QueryMasterTask extends CompositeService { } public synchronized void startQuery() { - if(query != null) { - LOG.warn("Query already started"); - return; - } - CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); - LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); - Expr expr; - if (queryContext.isHiveQueryMode()) { - HiveQLAnalyzer HiveQLAnalyzer = new HiveQLAnalyzer(); - expr = HiveQLAnalyzer.parse(sql); - } else { - SQLAnalyzer analyzer = new SQLAnalyzer(); - expr = analyzer.parse(sql); - } - LogicalPlan plan = null; try { - plan = planner.createPlan(session, expr); + if (query != null) { + LOG.warn("Query already started"); + return; + } + CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); + LogicalPlanner planner = new LogicalPlanner(catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); + Expr expr; + if (queryContext.isHiveQueryMode()) { + HiveQLAnalyzer HiveQLAnalyzer = new HiveQLAnalyzer(); + expr = HiveQLAnalyzer.parse(sql); + } else { + SQLAnalyzer analyzer = new SQLAnalyzer(); + expr = analyzer.parse(sql); + } + LogicalPlan plan = planner.createPlan(session, expr); optimizer.optimize(plan); - } catch (PlanningException e) { - //TODO how set query failed(???) - LOG.error(e.getMessage(), e); - } - - GlobalEngine.DistributedQueryHookManager hookManager = new GlobalEngine.DistributedQueryHookManager(); - hookManager.addHook(new GlobalEngine.InsertHook()); - hookManager.doHooks(queryContext, plan); - try { + GlobalEngine.DistributedQueryHookManager hookManager = new GlobalEngine.DistributedQueryHookManager(); + hookManager.addHook(new GlobalEngine.InsertHook()); + hookManager.doHooks(queryContext, plan); for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { LogicalNode[] scanNodes = PlannerUtil.findAllNodes(block.getRoot(), NodeType.SCAN); - if(scanNodes != null) { - for(LogicalNode eachScanNode: scanNodes) { - ScanNode scanNode = (ScanNode)eachScanNode; + if (scanNodes != null) { + for (LogicalNode eachScanNode : scanNodes) { + ScanNode scanNode = (ScanNode) eachScanNode; tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); } } scanNodes = PlannerUtil.findAllNodes(block.getRoot(), NodeType.PARTITIONS_SCAN); - if(scanNodes != null) { - for(LogicalNode eachScanNode: scanNodes) { - ScanNode scanNode = (ScanNode)eachScanNode; + if (scanNodes != null) { + for (LogicalNode eachScanNode : scanNodes) { + ScanNode scanNode = (ScanNode) eachScanNode; tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); } } @@ -349,11 +346,9 @@ public class QueryMasterTask extends CompositeService { dispatcher.register(QueryEventType.class, query); queryTaskContext.getEventHandler().handle(new QueryEvent(queryId, QueryEventType.START)); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - //TODO how set query failed(???) - //send FAIL query status - //this.statusMessage = StringUtils.stringifyException(e); + } catch (Throwable t) { + LOG.error(t.getMessage(), t); + initError = t; } } @@ -471,14 +466,34 @@ public class QueryMasterTask extends CompositeService { return queryId; } + public boolean isInitError() { + return initError != null; + } + public QueryState getState() { if(query == null) { - return QueryState.QUERY_NOT_ASSIGNED; + if (isInitError()) { + return QueryState.QUERY_ERROR; + } else { + return QueryState.QUERY_NOT_ASSIGNED; + } } else { return query.getState(); } } + public String getErrorMessage() { + if (isInitError()) { + return StringUtils.stringifyException(initError); + } else { + return null; + } + } + + public long getQuerySubmitTime() { + return this.querySubmitTime; + } + public class QueryMasterTaskContext { EventHandler eventHandler; public QueryMaster.QueryMasterContext getQueryMasterContext() { http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java index a73623f..937d886 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java @@ -184,20 +184,26 @@ public class TajoWorkerClientService extends AbstractService { return builder.build(); } - queryMasterTask.touchSessionTime(); - Query query = queryMasterTask.getQuery(); - - builder.setState(query.getState()); - builder.setProgress(query.getProgress()); - builder.setSubmitTime(query.getAppSubmitTime()); builder.setHasResult( !(queryMasterTask.getQueryTaskContext().getQueryContext().isCreateTable() || queryMasterTask.getQueryTaskContext().getQueryContext().isInsert()) ); - if (query.getState() == TajoProtos.QueryState.QUERY_SUCCEEDED) { - builder.setFinishTime(query.getFinishTime()); + + queryMasterTask.touchSessionTime(); + Query query = queryMasterTask.getQuery(); + + if (query != null) { + builder.setState(query.getState()); + builder.setProgress(query.getProgress()); + builder.setSubmitTime(query.getAppSubmitTime()); + if (query.getState() == TajoProtos.QueryState.QUERY_SUCCEEDED) { + builder.setFinishTime(query.getFinishTime()); + } else { + builder.setFinishTime(System.currentTimeMillis()); + } } else { - builder.setFinishTime(System.currentTimeMillis()); + builder.setState(queryMasterTask.getState()); + builder.setErrorMessage(queryMasterTask.getErrorMessage()); } } return builder.build(); http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp b/tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp index 2e64f15..42e0b9d 100644 --- a/tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp +++ b/tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp @@ -107,7 +107,7 @@ QueryIdQuery MasterStartedFinishedTimeStatussql <% for(QueryInProgress eachQuery: finishedQueries) { - long runTime = eachQuery.getQueryInfo().getFinishTime() >= 0 ? + long runTime = eachQuery.getQueryInfo().getFinishTime() > 0 ? eachQuery.getQueryInfo().getFinishTime() - eachQuery.getQueryInfo().getStartTime() : -1; String detailView = "http://" + eachQuery.getQueryInfo().getQueryMasterHost() + ":" + portMap.get(eachQuery.getQueryInfo().getQueryMasterHost()) + "/querydetail.jsp?queryId=" + eachQuery.getQueryId(); @@ -116,8 +116,8 @@ <%=eachQuery.getQueryId()%> <%=eachQuery.getQueryInfo().getQueryMasterHost()%> <%=df.format(eachQuery.getQueryInfo().getStartTime())%> - <%=eachQuery.getQueryInfo().getFinishTime() >= 0 ? df.format(eachQuery.getQueryInfo().getFinishTime()) : "N/A"%> - <%=runTime == -1 ? "N/A" : StringUtils.formatTime(runTime) %> + <%=eachQuery.getQueryInfo().getFinishTime() > 0 ? df.format(eachQuery.getQueryInfo().getFinishTime()) : "-"%> + <%=runTime == -1 ? "-" : StringUtils.formatTime(runTime) %> <%=eachQuery.getQueryInfo().getQueryState()%> <%=eachQuery.getQueryInfo().getSql()%> http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp b/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp index 1150ade..c30a72d 100644 --- a/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp +++ b/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp @@ -70,13 +70,14 @@ if(tajoWorker.getWorkerContext().isQueryMasterMode()) { } else { %> - + <% for(QueryMasterTask eachQueryMasterTask: queryMasterTasks) { Query query = eachQueryMasterTask.getQuery(); %> + @@ -96,17 +97,19 @@ if(tajoWorker.getWorkerContext().isQueryMasterMode()) { } else { %>
QueryIdStartTimeFinishTimeProgressRunTime
QueryIdStatusStartTimeFinishTimeProgressRunTime
<%=query.getId()%><%=eachQueryMasterTask.getState()%> <%=df.format(query.getStartTime())%> <%=query.getFinishTime() == 0 ? "-" : df.format(query.getFinishTime())%> <%=(int)(query.getProgress()*100.0f)%>%
- + <% for(QueryMasterTask eachQueryMasterTask: finishedQueryMasterTasks) { Query query = eachQueryMasterTask.getQuery(); + long startTime = query != null ? query.getStartTime() : eachQueryMasterTask.getQuerySubmitTime(); %> - - - - - + + + + + + <% } //end of for http://git-wip-us.apache.org/repos/asf/tajo/blob/6e50b57c/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp b/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp index 2d867ed..3de20fe 100644 --- a/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp +++ b/tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp @@ -34,12 +34,15 @@ QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext() .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true); - if(queryMasterTask == null) { + if (queryMasterTask == null) { out.write(""); return; } Query query = queryMasterTask.getQuery(); - List subQueries = JSPUtil.sortSubQuery(query.getSubQueries()); + List subQueries = null; + if (query != null) { + subQueries = JSPUtil.sortSubQuery(query.getSubQueries()); + } SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); %> @@ -56,8 +59,16 @@

Tajo Worker: <%=tajoWorker.getWorkerContext().getWorkerName()%>

+<% +if (query == null) { + String errorMessage = queryMasterTask.getErrorMessage(); + out.write("Query Status: " + queryMasterTask.getState()); + if (errorMessage != null && !errorMessage.isEmpty()) { + out.write("

Message:

" + errorMessage + "
"); + } +} else { +%>

<%=queryId.toString()%> [Query Plan]

-
QueryIdStartTimeFinishTimeProgressRunTime
QueryIdStatusStartTimeFinishTimeProgressRunTime
<%=query.getId()%><%=df.format(query.getStartTime())%><%=query.getFinishTime() == 0 ? "-" : df.format(query.getFinishTime())%><%=(int)(query.getProgress()*100.0f)%>%<%=JSPUtil.getElapsedTime(query.getStartTime(), query.getFinishTime())%><%=eachQueryMasterTask.getQueryId()%><%=eachQueryMasterTask.getState()%><%=df.format(startTime)%><%=(query == null || query.getFinishTime() == 0) ? "-" : df.format(query.getFinishTime())%><%=(query == null) ? "-" : (int)(query.getProgress()*100.0f)%>%<%=(query == null) ? "-" : JSPUtil.getElapsedTime(query.getStartTime(), query.getFinishTime())%>
<% @@ -86,6 +97,9 @@ for(SubQuery eachSubQuery: subQueries) {

Distributed Query Plan

<%=query.getPlan().toString()%>
+<% +} //end of else [if (query == null)] +%> \ No newline at end of file
IDStateStartedFinishedRunning timeProgressTasks