tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [4/4] tajo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Date Tue, 24 Mar 2015 08:39:53 GMT
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support

Conflicts:
	tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
	tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/12846cca
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/12846cca
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/12846cca

Branch: refs/heads/index_support
Commit: 12846ccae5bd5c3673d8812d53010ef432c947c6
Parents: d09bd8d 3e9a2dd
Author: Jihoon Son <jihoonson@apache.org>
Authored: Tue Mar 24 17:39:46 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Tue Mar 24 17:39:46 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |   9 +
 .../java/org/apache/tajo/algebra/Explain.java   |   9 +-
 .../main/java/org/apache/tajo/SessionVars.java  |   1 +
 .../java/org/apache/tajo/conf/TajoConf.java     |   1 +
 .../org/apache/tajo/datum/BooleanDatum.java     |   9 +-
 .../org/apache/tajo/datum/DatumFactory.java     |   4 +
 .../java/org/apache/tajo/datum/TextDatum.java   |   4 +-
 .../java/org/apache/tajo/util/FileUtil.java     |  25 ++-
 .../org/apache/tajo/engine/parser/SQLLexer.g4   |   1 +
 .../org/apache/tajo/engine/parser/SQLParser.g4  |   2 +-
 .../apache/tajo/engine/parser/SQLAnalyzer.java  |   2 +-
 .../engine/planner/global/GlobalPlanner.java    |   4 +
 .../engine/planner/physical/MemSortExec.java    |  28 ++-
 .../tajo/engine/planner/physical/SortExec.java  |   4 +-
 .../engine/planner/physical/TupleSorter.java    |  27 +++
 .../planner/physical/VectorizedSorter.java      | 198 +++++++++++++++++
 .../org/apache/tajo/master/QueryInProgress.java |  22 +-
 .../org/apache/tajo/master/QueryManager.java    |   8 +-
 .../ExplainGlobalPlanPreprocessorForTest.java   |  62 ++++++
 .../exec/ExplainPlanPreprocessorForTest.java    | 218 +++++++++++++++++++
 .../apache/tajo/master/exec/QueryExecutor.java  |  61 +++++-
 .../tajo/querymaster/QueryMasterTask.java       |   3 -
 .../java/org/apache/tajo/QueryTestCaseBase.java |  75 ++++++-
 .../planner/physical/TestTupleSorter.java       | 132 +++++++++++
 .../tajo/engine/query/TestSelectQuery.java      |  16 ++
 .../testExplainSelectPhysical.1.result          |  26 +++
 .../testExplainSelectPhysical.2.result          |  88 ++++++++
 .../testExplainSelectPhysical.3.result          |  89 ++++++++
 .../java/org/apache/tajo/plan/LogicalPlan.java  |  17 +-
 .../org/apache/tajo/plan/LogicalPlanner.java    |   2 +-
 .../apache/tajo/plan/logical/IndexScanNode.java |   4 +-
 31 files changed, 1093 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index 31a28f7,75e7762..ca3abbd
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@@ -48,6 -47,7 +50,8 @@@ import org.apache.tajo.master.*
  import org.apache.tajo.master.exec.prehook.CreateTableHook;
  import org.apache.tajo.master.exec.prehook.DistributedQueryHookManager;
  import org.apache.tajo.master.exec.prehook.InsertIntoHook;
+ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
++import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
  import org.apache.tajo.querymaster.*;
  import org.apache.tajo.session.Session;
  import org.apache.tajo.plan.LogicalPlan;
@@@ -99,20 -98,13 +103,20 @@@ public class QueryExecutor 
  
      } else if (PlannerUtil.checkIfDDLPlan(rootNode)) {
        context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
 -      ddlExecutor.execute(queryContext, plan);
 -      response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
 -      response.setResultCode(ClientProtos.ResultCode.OK);
  
 +      if (PlannerUtil.isDistExecDDL(rootNode)) {
 +        if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) {
 +          checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild());
 +        }
 +        executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
 +      } else {
 +        response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
 +        response.setResult(IPCUtil.buildOkRequestResult());
 +        ddlExecutor.execute(queryContext, plan);
 +      }
  
      } else if (plan.isExplain()) { // explain query
-       execExplain(plan, response);
+       execExplain(plan, queryContext, plan.isExplainGlobal(), response);
  
      } else if (PlannerUtil.checkIfQueryTargetIsVirtualTable(plan)) {
        execQueryOnVirtualTable(queryContext, session, sql, plan, response);
@@@ -165,12 -157,31 +169,31 @@@
  
      context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
      response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
 -    response.setResultCode(ClientProtos.ResultCode.OK);
 +    response.setResult(IPCUtil.buildOkRequestResult());
    }
  
-   public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) throws
IOException {
+   public void execExplain(LogicalPlan plan, QueryContext queryContext, boolean isGlobal,
+                           SubmitQueryResponse.Builder response)
+       throws Exception {
+     String explainStr;
+     boolean isTest = queryContext.getBool(SessionVars.TEST_PLAN_SHAPE_FIX_ENABLED);
+     if (isTest) {
+       ExplainPlanPreprocessorForTest preprocessorForTest = new ExplainPlanPreprocessorForTest();
+       preprocessorForTest.prepareTest(plan);
+     }
+ 
+     if (isGlobal) {
+       GlobalPlanner planner = new GlobalPlanner(context.getConf(), context.getCatalog());
+       MasterPlan masterPlan = compileMasterPlan(plan, queryContext, planner);
+       if (isTest) {
+         ExplainGlobalPlanPreprocessorForTest globalPlanPreprocessorForTest = new ExplainGlobalPlanPreprocessorForTest();
+         globalPlanPreprocessorForTest.prepareTest(masterPlan);
+       }
+       explainStr = masterPlan.toString();
+     } else {
+       explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
+     }
  
-     String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
      Schema schema = new Schema();
      schema.addColumn("explain", TajoDataTypes.Type.TEXT);
      RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
@@@ -428,22 -439,33 +451,52 @@@
      }
    }
  
 +  private void checkIndexExistence(final QueryContext queryContext, final CreateIndexNode
createIndexNode)
 +      throws IOException {
 +    String databaseName, simpleIndexName, qualifiedIndexName;
 +    if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) {
-       String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
++      String[] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
 +      databaseName = splits[0];
 +      simpleIndexName = splits[1];
 +      qualifiedIndexName = createIndexNode.getIndexName();
 +    } else {
 +      databaseName = queryContext.getCurrentDatabase();
 +      simpleIndexName = createIndexNode.getIndexName();
 +      qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName);
 +    }
 +
 +    if (catalog.existIndexByName(databaseName, simpleIndexName)) {
 +      throw new AlreadyExistsIndexException(qualifiedIndexName);
 +    }
 +  }
++
+   public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner
planner)
+       throws Exception {
+ 
+     CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan);
+     if (storeType != null) {
+       StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType);
+       StorageProperty storageProperty = sm.getStorageProperty();
+       if (storageProperty.isSortedInsert()) {
+         String tableName = PlannerUtil.getStoreTableName(plan);
+         LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+         TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());
+         if (tableDesc == null) {
+           throw new VerifyException("Can't get table meta data from catalog: " + tableName);
+         }
+         List<LogicalPlanRewriteRule> storageSpecifiedRewriteRules = sm.getRewriteRules(
+             context, tableDesc);
+         if (storageSpecifiedRewriteRules != null) {
+           for (LogicalPlanRewriteRule eachRule: storageSpecifiedRewriteRules) {
 -            eachRule.rewrite(context, plan);
++            eachRule.rewrite(new LogicalPlanRewriteRuleContext(context, plan));
+           }
+         }
+       }
+     }
+ 
+     MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
+     planner.build(masterPlan);
+ 
+     return masterPlan;
+   }
  }

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index d38e2da,f83cb1e..26111c3
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@@ -51,12 -46,15 +51,9 @@@ import org.apache.tajo.ipc.TajoWorkerPr
  import org.apache.tajo.master.TajoContainerProxy;
  import org.apache.tajo.master.event.*;
  import org.apache.tajo.master.rm.TajoWorkerResourceManager;
--import org.apache.tajo.plan.LogicalOptimizer;
--import org.apache.tajo.plan.LogicalPlan;
--import org.apache.tajo.plan.LogicalPlanner;
  import org.apache.tajo.plan.logical.LogicalNode;
 -import org.apache.tajo.plan.logical.LogicalRootNode;
  import org.apache.tajo.plan.logical.NodeType;
  import org.apache.tajo.plan.logical.ScanNode;
 -import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
 -import org.apache.tajo.plan.util.PlannerUtil;
  import org.apache.tajo.plan.verifier.VerifyException;
  import org.apache.tajo.session.Session;
  import org.apache.tajo.storage.StorageManager;

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
index e320ce9,8b73756..ebf6c4b
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
@@@ -82,9 -84,9 +82,9 @@@ public class IndexScanNode extends Scan
    public int hashCode() {
      final int prime = 31;
      int result = super.hashCode();
--    result = prime * result + Arrays.hashCode(datum);
++    result = prime * result + indexPath.hashCode();
      result = prime * result + ((keySchema == null) ? 0 : keySchema.hashCode());
--    result = prime * result + Arrays.hashCode(sortKeys);
++    result = prime * result + Arrays.hashCode(predicates);
      return result;
    }
  


Mime
View raw message