tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [5/5] tajo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Date Fri, 24 Jul 2015 03:39:06 GMT
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support

Conflicts:
	CHANGES


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1bf1536d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1bf1536d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1bf1536d

Branch: refs/heads/index_support
Commit: 1bf1536db45b50f80872fe01ac8f1b4a2400f0d2
Parents: 5a8fed4 bcec5eb
Author: Jihoon Son <jihoonson@apache.org>
Authored: Fri Jul 24 12:38:56 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Fri Jul 24 12:38:56 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  10 ++
 .../apache/tajo/client/SessionConnection.java   |  17 ++-
 .../main/proto/TajoMasterClientProtocol.proto   |   2 +-
 .../planner/physical/ColPartitionStoreExec.java |  17 +--
 .../HashBasedColPartitionStoreExec.java         |  59 ++++-----
 .../SortBasedColPartitionStoreExec.java         |  60 ++++-----
 .../tajo/master/TajoMasterClientService.java    | 105 +++++++++++----
 .../tajo/engine/query/TestInnerJoinQuery.java   |   1 -
 .../engine/query/TestInnerJoinWithSubQuery.java |   1 -
 .../engine/query/TestMultipleJoinTypes.java     |   1 -
 .../tajo/engine/query/TestOuterJoinQuery.java   |   2 -
 ...sJoinWithThetaJoinConditionInWhere.Hash.plan |   8 +-
 ...taJoinConditionInWhere.Hash_NoBroadcast.plan |  12 +-
 ...sJoinWithThetaJoinConditionInWhere.Sort.plan |   8 +-
 ...taJoinConditionInWhere.Sort_NoBroadcast.plan |  12 +-
 ...rJoinWithThetaJoinConditionInWhere.Hash.plan |   8 +-
 ...taJoinConditionInWhere.Hash_NoBroadcast.plan |  12 +-
 ...rJoinWithThetaJoinConditionInWhere.Sort.plan |   8 +-
 ...taJoinConditionInWhere.Sort_NoBroadcast.plan |  12 +-
 .../testBroadcastSubquery2.Hash.plan            |  28 ++--
 ...testBroadcastSubquery2.Hash_NoBroadcast.plan |  44 +++----
 .../testBroadcastSubquery2.Sort.plan            |  28 ++--
 ...testBroadcastSubquery2.Sort_NoBroadcast.plan |  44 +++----
 .../testJoinWithMultipleJoinQual4.Hash.plan     |  16 +--
 ...nWithMultipleJoinQual4.Hash_NoBroadcast.plan |  28 ++--
 .../testJoinWithMultipleJoinQual4.Sort.plan     |  16 +--
 ...nWithMultipleJoinQual4.Sort_NoBroadcast.plan |  28 ++--
 .../testThetaJoinKeyPairs.Hash.plan             | 128 ++++++++++---------
 .../testThetaJoinKeyPairs.Hash_NoBroadcast.plan | 114 +++++++++--------
 .../testThetaJoinKeyPairs.Sort.plan             | 128 ++++++++++---------
 .../testThetaJoinKeyPairs.Sort_NoBroadcast.plan | 114 +++++++++--------
 .../testThetaJoinKeyPairs.result                |   6 -
 ...rJoinWithThetaJoinConditionInWhere.Hash.plan |   8 +-
 ...taJoinConditionInWhere.Hash_NoBroadcast.plan |  12 +-
 ...rJoinWithThetaJoinConditionInWhere.Sort.plan |   8 +-
 ...taJoinConditionInWhere.Sort_NoBroadcast.plan |  12 +-
 ...rJoinWithThetaJoinConditionInWhere.Hash.plan |   8 +-
 ...taJoinConditionInWhere.Hash_NoBroadcast.plan |  12 +-
 ...rJoinWithThetaJoinConditionInWhere.Sort.plan |   8 +-
 ...taJoinConditionInWhere.Sort_NoBroadcast.plan |  12 +-
 .../org/apache/tajo/plan/LogicalOptimizer.java  |  13 +-
 .../org/apache/tajo/plan/expr/EvalTreeUtil.java |   2 +-
 .../plan/rewrite/rules/FilterPushDownRule.java  |  55 +++++++-
 43 files changed, 669 insertions(+), 558 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 9fb7eb3,420aa4b..2c32817
--- a/CHANGES
+++ b/CHANGES
@@@ -188,13 -191,8 +191,15 @@@ Release 0.11.0 - unrelease
  
    BUG FIXES
  
 +    TAJO-1608: Fix test failure in index_support branch. (jihoon)
 +
 +    TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
 +
 +    TAJO-1657: Tajo Rest API /database/{database-name]/tables should return table 
 +    names only without invalid external table info. 
 +
+     TAJO-1597: Problem of ignoring theta join condition. (jihoon)
+ 
      TAJO-1697: RCFile progress causes NPE occasionally. (jinho)
  
      TAJO-1689: Metrics file reporter prints histogram metric without group name.
@@@ -406,7 -404,8 +411,12 @@@
  
    SUB TASKS
  
++<<<<<<< HEAD
 +    TAJO-1302: Support index metadata backup and restore. (jihoon)
++=======
+     TAJO-1484 Apply on ColPartitionStoreExec. (Contributed by Navis, 
+     committed by hyunsik)
++>>>>>>> bcec5eb7e247dc983184b37a3dbb4143afd19ac8
  
      TAJO-1464: Add ORCFileScanner to read ORCFile table. (Contributed by 
      Jongyoung Park, Committed by jihoon)

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index c1afbc9,baf1320..0e9b6e6
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@@ -61,14 -65,15 +64,18 @@@ import org.apache.tajo.util.ProtoUtil
  
  import java.io.IOException;
  import java.net.InetSocketAddress;
 -import java.util.*;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.List;
  
  import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
+ import static org.apache.tajo.exception.ExceptionUtil.printStackTraceIfError;
  import static org.apache.tajo.exception.ReturnStateUtil.*;
  
+ /**
+  * It provides Client Remote API service for TajoMaster.
+  */
  public class TajoMasterClientService extends AbstractService {
    private final static Log LOG = LogFactory.getLog(TajoMasterClientService.class);
    private final MasterContext context;
@@@ -899,203 -953,12 +954,207 @@@
              .build();
  
        } catch (Throwable t) {
-         return FunctionListResponse.newBuilder().setState(returnError(t)).build();
+         printStackTraceIfError(LOG, t);
+ 
+         return FunctionListResponse.newBuilder().
+             setState(returnError(t))
+             .build();
        }
      }
 +
 +    @Override
 +    public IndexResponse getIndexWithName(RpcController controller, SessionedStringProto
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String indexName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getValue())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
 +          databaseName = splitted[0];
 +          indexName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          indexName = request.getValue();
 +        }
 +        IndexDescProto indexProto = catalog.getIndexByName(databaseName, indexName).getProto();
 +        return IndexResponse.newBuilder()
 +            .setState(OK)
 +            .setIndexDesc(indexProto)
 +            .build();
 +
 +      } catch (Throwable t) {
 +        return IndexResponse.newBuilder()
 +            .setState(returnError(t))
 +            .build();
 +      }
 +    }
 +
 +    @Override
 +    public ReturnState existIndexWithName(RpcController controller, SessionedStringProto
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String indexName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getValue())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
 +          databaseName = splitted[0];
 +          indexName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          indexName = request.getValue();
 +        }
 +
 +        if (catalog.existIndexByName(databaseName, indexName)) {
 +          return OK;
 +        } else {
 +          return errUndefinedIndexName(indexName);
 +        }
 +      } catch (Throwable t) {
 +        return returnError(t);
 +      }
 +    }
 +
 +    @Override
 +    public IndexListResponse getIndexesForTable(RpcController controller, SessionedStringProto
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String tableName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getValue())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
 +          databaseName = splitted[0];
 +          tableName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          tableName = request.getValue();
 +        }
 +
 +        IndexListResponse.Builder builder = IndexListResponse.newBuilder().setState(OK);
 +        for (IndexDesc index : catalog.getAllIndexesByTable(databaseName, tableName)) {
 +          builder.addIndexDesc(index.getProto());
 +        }
 +        return builder.build();
 +      } catch (Throwable t) {
 +        return IndexListResponse.newBuilder()
 +            .setState(returnError(t))
 +            .build();
 +      }
 +    }
 +
 +    @Override
 +    public ReturnState existIndexesForTable(RpcController controller, SessionedStringProto
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String tableName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getValue())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
 +          databaseName = splitted[0];
 +          tableName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          tableName = request.getValue();
 +        }
 +        if (catalog.existIndexesByTable(databaseName, tableName)) {
 +          return OK;
 +        } else {
 +          return errUndefinedIndex(tableName);
 +        }
 +      } catch (Throwable t) {
 +        return returnError(t);
 +      }
 +    }
 +
 +    @Override
 +    public IndexResponse getIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String tableName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getTableName())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getTableName());
 +          databaseName = splitted[0];
 +          tableName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          tableName = request.getTableName();
 +        }
 +        String[] columnNames = new String[request.getColumnNamesCount()];
 +        columnNames = request.getColumnNamesList().toArray(columnNames);
 +
 +        return IndexResponse.newBuilder()
 +            .setState(OK)
 +            .setIndexDesc(catalog.getIndexByColumnNames(databaseName, tableName, columnNames).getProto())
 +            .build();
 +
 +      } catch (Throwable t) {
 +        return IndexResponse.newBuilder()
 +            .setState(returnError(t))
 +            .build();
 +      }
 +    }
 +
 +    @Override
 +    public ReturnState existIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest
request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +
 +        String tableName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getTableName())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getTableName());
 +          databaseName = splitted[0];
 +          tableName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          tableName = request.getTableName();
 +        }
 +        String[] columnNames = new String[request.getColumnNamesCount()];
 +        columnNames = request.getColumnNamesList().toArray(columnNames);
 +        if (catalog.existIndexByColumnNames(databaseName, tableName, columnNames)) {
 +          return OK;
 +        } else {
 +          return errUndefinedIndex(tableName, request.getColumnNamesList());
 +        }
 +      } catch (Throwable t) {
 +        return returnError(t);
 +      }
 +    }
 +
 +    @Override
 +    public ReturnState dropIndex(RpcController controller, SessionedStringProto request)
 +        throws ServiceException {
 +      try {
 +        context.getSessionManager().touch(request.getSessionId().getId());
 +        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
 +        QueryContext queryContext = new QueryContext(conf, session);
 +
 +        String indexName, databaseName;
 +        if (CatalogUtil.isFQTableName(request.getValue())) {
 +          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
 +          databaseName = splitted[0];
 +          indexName = splitted[1];
 +        } else {
 +          databaseName = session.getCurrentDatabase();
 +          indexName = request.getValue();
 +        }
 +        catalog.dropIndex(databaseName, indexName);
 +
 +        return OK;
 +      } catch (Throwable t) {
 +        return returnError(t);
 +      }
 +    }
    }
  }

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index ebc35a5,98e674e..4fb8aac
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@@ -35,12 -32,10 +35,13 @@@ import org.apache.tajo.catalog.TableDes
  import org.apache.tajo.exception.TajoException;
  import org.apache.tajo.exception.TajoInternalError;
  import org.apache.tajo.plan.*;
+ import org.apache.tajo.plan.LogicalPlan.QueryBlock;
  import org.apache.tajo.plan.expr.*;
  import org.apache.tajo.plan.logical.*;
 +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
  import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext;
 +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
 +import org.apache.tajo.plan.util.IndexUtil;
  import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
  import org.apache.tajo.plan.util.PlannerUtil;
  import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;


Mime
View raw message