Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5E48718521 for ; Fri, 24 Jul 2015 03:39:03 +0000 (UTC) Received: (qmail 65686 invoked by uid 500); 24 Jul 2015 03:39:03 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 65597 invoked by uid 500); 24 Jul 2015 03:39:03 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 65460 invoked by uid 99); 24 Jul 2015 03:39:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Jul 2015 03:39:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A5CFFDFC11; Fri, 24 Jul 2015 03:39:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jihoonson@apache.org To: commits@tajo.apache.org Date: Fri, 24 Jul 2015 03:39:06 -0000 Message-Id: In-Reply-To: <4ac812d1931f44cea299930d58e391ef@git.apache.org> References: <4ac812d1931f44cea299930d58e391ef@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [5/5] tajo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support Conflicts: CHANGES Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1bf1536d Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1bf1536d Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1bf1536d Branch: refs/heads/index_support Commit: 1bf1536db45b50f80872fe01ac8f1b4a2400f0d2 Parents: 5a8fed4 bcec5eb Author: Jihoon Son Authored: Fri Jul 24 12:38:56 2015 +0900 Committer: Jihoon Son Committed: Fri Jul 24 12:38:56 2015 +0900 ---------------------------------------------------------------------- CHANGES | 10 ++ .../apache/tajo/client/SessionConnection.java | 17 ++- .../main/proto/TajoMasterClientProtocol.proto | 2 +- .../planner/physical/ColPartitionStoreExec.java | 17 +-- .../HashBasedColPartitionStoreExec.java | 59 ++++----- .../SortBasedColPartitionStoreExec.java | 60 ++++----- .../tajo/master/TajoMasterClientService.java | 105 +++++++++++---- .../tajo/engine/query/TestInnerJoinQuery.java | 1 - .../engine/query/TestInnerJoinWithSubQuery.java | 1 - .../engine/query/TestMultipleJoinTypes.java | 1 - .../tajo/engine/query/TestOuterJoinQuery.java | 2 - ...sJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +- ...sJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +- ...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +- ...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +- .../testBroadcastSubquery2.Hash.plan | 28 ++-- ...testBroadcastSubquery2.Hash_NoBroadcast.plan | 44 +++---- .../testBroadcastSubquery2.Sort.plan | 28 ++-- ...testBroadcastSubquery2.Sort_NoBroadcast.plan | 44 +++---- .../testJoinWithMultipleJoinQual4.Hash.plan | 16 +-- ...nWithMultipleJoinQual4.Hash_NoBroadcast.plan | 28 ++-- .../testJoinWithMultipleJoinQual4.Sort.plan | 16 +-- ...nWithMultipleJoinQual4.Sort_NoBroadcast.plan | 28 ++-- .../testThetaJoinKeyPairs.Hash.plan | 128 ++++++++++--------- .../testThetaJoinKeyPairs.Hash_NoBroadcast.plan | 114 +++++++++-------- .../testThetaJoinKeyPairs.Sort.plan | 128 ++++++++++--------- .../testThetaJoinKeyPairs.Sort_NoBroadcast.plan | 114 +++++++++-------- .../testThetaJoinKeyPairs.result | 6 - ...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +- ...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +- ...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +- ...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +- .../org/apache/tajo/plan/LogicalOptimizer.java | 13 +- .../org/apache/tajo/plan/expr/EvalTreeUtil.java | 2 +- .../plan/rewrite/rules/FilterPushDownRule.java | 55 +++++++- 43 files changed, 669 insertions(+), 558 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index 9fb7eb3,420aa4b..2c32817 --- a/CHANGES +++ b/CHANGES @@@ -188,13 -191,8 +191,15 @@@ Release 0.11.0 - unrelease BUG FIXES + TAJO-1608: Fix test failure in index_support branch. (jihoon) + + TAJO-1594: Catalog schema is invalid for some databases. (jihoon) + + TAJO-1657: Tajo Rest API /database/{database-name]/tables should return table + names only without invalid external table info. + + TAJO-1597: Problem of ignoring theta join condition. (jihoon) + TAJO-1697: RCFile progress causes NPE occasionally. (jinho) TAJO-1689: Metrics file reporter prints histogram metric without group name. @@@ -406,7 -404,8 +411,12 @@@ SUB TASKS ++<<<<<<< HEAD + TAJO-1302: Support index metadata backup and restore. (jihoon) ++======= + TAJO-1484 Apply on ColPartitionStoreExec. (Contributed by Navis, + committed by hyunsik) ++>>>>>>> bcec5eb7e247dc983184b37a3dbb4143afd19ac8 TAJO-1464: Add ORCFileScanner to read ORCFile table. (Contributed by Jongyoung Park, Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/proto/TajoMasterClientProtocol.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index c1afbc9,baf1320..0e9b6e6 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@@ -61,14 -65,15 +64,18 @@@ import org.apache.tajo.util.ProtoUtil import java.io.IOException; import java.net.InetSocketAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; + import static org.apache.tajo.exception.ExceptionUtil.printStackTraceIfError; import static org.apache.tajo.exception.ReturnStateUtil.*; + /** + * It provides Client Remote API service for TajoMaster. + */ public class TajoMasterClientService extends AbstractService { private final static Log LOG = LogFactory.getLog(TajoMasterClientService.class); private final MasterContext context; @@@ -899,203 -953,12 +954,207 @@@ .build(); } catch (Throwable t) { - return FunctionListResponse.newBuilder().setState(returnError(t)).build(); + printStackTraceIfError(LOG, t); + + return FunctionListResponse.newBuilder(). + setState(returnError(t)) + .build(); } } + + @Override + public IndexResponse getIndexWithName(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + IndexDescProto indexProto = catalog.getIndexByName(databaseName, indexName).getProto(); + return IndexResponse.newBuilder() + .setState(OK) + .setIndexDesc(indexProto) + .build(); + + } catch (Throwable t) { + return IndexResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + } + + @Override + public ReturnState existIndexWithName(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + + if (catalog.existIndexByName(databaseName, indexName)) { + return OK; + } else { + return errUndefinedIndexName(indexName); + } + } catch (Throwable t) { + return returnError(t); + } + } + + @Override + public IndexListResponse getIndexesForTable(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getValue(); + } + + IndexListResponse.Builder builder = IndexListResponse.newBuilder().setState(OK); + for (IndexDesc index : catalog.getAllIndexesByTable(databaseName, tableName)) { + builder.addIndexDesc(index.getProto()); + } + return builder.build(); + } catch (Throwable t) { + return IndexListResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + } + + @Override + public ReturnState existIndexesForTable(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getValue(); + } + if (catalog.existIndexesByTable(databaseName, tableName)) { + return OK; + } else { + return errUndefinedIndex(tableName); + } + } catch (Throwable t) { + return returnError(t); + } + } + + @Override + public IndexResponse getIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getTableName())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getTableName(); + } + String[] columnNames = new String[request.getColumnNamesCount()]; + columnNames = request.getColumnNamesList().toArray(columnNames); + + return IndexResponse.newBuilder() + .setState(OK) + .setIndexDesc(catalog.getIndexByColumnNames(databaseName, tableName, columnNames).getProto()) + .build(); + + } catch (Throwable t) { + return IndexResponse.newBuilder() + .setState(returnError(t)) + .build(); + } + } + + @Override + public ReturnState existIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getTableName())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getTableName(); + } + String[] columnNames = new String[request.getColumnNamesCount()]; + columnNames = request.getColumnNamesList().toArray(columnNames); + if (catalog.existIndexByColumnNames(databaseName, tableName, columnNames)) { + return OK; + } else { + return errUndefinedIndex(tableName, request.getColumnNamesList()); + } + } catch (Throwable t) { + return returnError(t); + } + } + + @Override + public ReturnState dropIndex(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + QueryContext queryContext = new QueryContext(conf, session); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + catalog.dropIndex(databaseName, indexName); + + return OK; + } catch (Throwable t) { + return returnError(t); + } + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index ebc35a5,98e674e..4fb8aac --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@@ -35,12 -32,10 +35,13 @@@ import org.apache.tajo.catalog.TableDes import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.plan.*; + import org.apache.tajo.plan.LogicalPlan.QueryBlock; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; +import org.apache.tajo.plan.util.IndexUtil; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;