tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [10/10] tajo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Date Sat, 16 May 2015 12:15:41 GMT
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2cbc1b9c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2cbc1b9c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2cbc1b9c

Branch: refs/heads/index_support
Commit: 2cbc1b9c7dc1ec04c4f6c8d7c8f746225d02421d
Parents: 6f44096 5491f0e
Author: Jihoon Son <jihoonson@apache.org>
Authored: Sat May 16 21:05:31 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Sat May 16 21:05:31 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  15 +
 .../java/org/apache/tajo/catalog/Schema.java    |  16 +
 .../java/org/apache/tajo/catalog/TableMeta.java |   6 +-
 .../org/apache/tajo/catalog/TestTableMeta.java  |  27 +
 .../org/apache/tajo/cli/tools/TajoAdmin.java    |   6 +-
 .../org/apache/tajo/cli/tools/TajoHAAdmin.java  |   9 +-
 .../cli/tsql/commands/DescTableCommand.java     |   1 +
 .../apache/tajo/client/DummyServiceTracker.java |  19 +-
 .../org/apache/tajo/storage/RowStoreUtil.java   |  24 +-
 .../java/org/apache/tajo/conf/TajoConf.java     |   2 +
 .../ValueTooLongForTypeCharactersException.java |  27 +
 .../java/org/apache/tajo/ha/HAConstants.java    |   1 +
 .../java/org/apache/tajo/ha/HAServiceUtil.java  | 253 --------
 .../apache/tajo/service/BaseServiceTracker.java |  31 +-
 .../apache/tajo/service/HAServiceTracker.java   |  25 +-
 .../org/apache/tajo/service/ServiceTracker.java |  28 +-
 .../org/apache/tajo/storage/EmptyTuple.java     | 140 +---
 .../java/org/apache/tajo/storage/NullTuple.java | 175 +++++
 .../java/org/apache/tajo/storage/VTuple.java    |  20 +-
 .../engine/planner/PhysicalPlannerImpl.java     |  25 +-
 .../planner/physical/BSTIndexScanExec.java      |   4 +-
 .../physical/BasicPhysicalExecutorVisitor.java  |   8 -
 .../planner/physical/ColPartitionStoreExec.java |   7 +-
 .../planner/physical/CommonHashJoinExec.java    | 191 ++++++
 .../engine/planner/physical/CommonJoinExec.java | 172 ++++-
 .../planner/physical/HashFullOuterJoinExec.java | 247 +++----
 .../engine/planner/physical/HashJoinExec.java   | 212 +-----
 .../planner/physical/HashLeftAntiJoinExec.java  |  59 +-
 .../planner/physical/HashLeftOuterJoinExec.java | 292 +--------
 .../planner/physical/HashLeftSemiJoinExec.java  |  48 +-
 .../planner/physical/NLLeftOuterJoinExec.java   | 101 ---
 .../physical/PhysicalExecutorVisitor.java       |   3 -
 .../physical/RangeShuffleFileWriteExec.java     |   3 +-
 .../physical/RightOuterMergeJoinExec.java       |  40 +-
 .../engine/planner/physical/SeqScanExec.java    |   2 +-
 .../engine/planner/physical/StoreTableExec.java |   9 +-
 .../apache/tajo/engine/utils/CacheHolder.java   |   3 +-
 .../org/apache/tajo/ha/HdfsServiceTracker.java  | 322 +++++----
 .../org/apache/tajo/master/GlobalEngine.java    |   4 +-
 .../java/org/apache/tajo/master/TajoMaster.java |  24 +-
 .../apache/tajo/master/exec/DDLExecutor.java    |   5 +-
 .../exec/NonForwardQueryResultFileScanner.java  |   7 +-
 .../apache/tajo/master/exec/QueryExecutor.java  |   4 +-
 .../java/org/apache/tajo/querymaster/Query.java |   7 +-
 .../tajo/querymaster/QueryMasterTask.java       |   4 +-
 .../apache/tajo/querymaster/Repartitioner.java  |  24 +-
 .../java/org/apache/tajo/querymaster/Stage.java |   6 +-
 .../main/java/org/apache/tajo/util/JSPUtil.java |   2 +-
 .../java/org/apache/tajo/worker/TajoWorker.java |   6 +-
 .../main/java/org/apache/tajo/worker/Task.java  |   3 +-
 .../tajo/worker/WorkerHeartbeatService.java     |   2 +-
 .../resources/webapps/admin/catalogview.jsp     |  11 +-
 .../main/resources/webapps/admin/cluster.jsp    |  10 +-
 .../src/main/resources/webapps/admin/index.jsp  |  10 +-
 .../src/main/resources/webapps/admin/query.jsp  |   7 +-
 .../resources/webapps/admin/query_executor.jsp  |   9 +-
 .../org/apache/tajo/BackendTestingUtil.java     |   3 +-
 .../planner/global/TestBroadcastJoinPlan.java   |   3 +-
 .../planner/physical/TestBNLJoinExec.java       |   5 +-
 .../planner/physical/TestExternalSortExec.java  |   3 +-
 .../physical/TestFullOuterHashJoinExec.java     |   9 +-
 .../physical/TestFullOuterMergeJoinExec.java    |  10 +-
 .../planner/physical/TestHashAntiJoinExec.java  |   5 +-
 .../planner/physical/TestHashJoinExec.java      |   5 +-
 .../planner/physical/TestHashSemiJoinExec.java  |  13 +-
 .../physical/TestLeftOuterHashJoinExec.java     | 113 ++--
 .../physical/TestLeftOuterNLJoinExec.java       | 474 --------------
 .../planner/physical/TestMergeJoinExec.java     |   5 +-
 .../engine/planner/physical/TestNLJoinExec.java |   5 +-
 .../planner/physical/TestPhysicalPlanner.java   |  11 +-
 .../physical/TestProgressExternalSortExec.java  |   3 +-
 .../physical/TestRightOuterHashJoinExec.java    |   7 +-
 .../physical/TestRightOuterMergeJoinExec.java   |  11 +-
 .../engine/planner/physical/TestSortExec.java   |   5 +-
 .../tajo/engine/query/TestHBaseTable.java       |  12 +-
 .../tajo/engine/query/TestInsertQuery.java      |  45 +-
 .../tajo/engine/query/TestJoinBroadcast.java    |   2 +-
 .../apache/tajo/engine/util/TestTupleUtil.java  |  18 +
 .../apache/tajo/ha/TestHAServiceHDFSImpl.java   |  28 +-
 .../org/apache/tajo/jdbc/TestResultSet.java     |   2 +-
 .../tajo/master/TestExecutionBlockCursor.java   |   4 +-
 .../org/apache/tajo/storage/TestRowFile.java    |   5 +-
 .../queries/TestInsertQuery/test1_ddl.sql       |   1 +
 .../TestInsertQuery/test1_nolength_ddl.sql      |   1 +
 .../testInsertIntoSelectWithFixedSizeChar.sql   |   4 +
 ...tIntoSelectWithFixedSizeCharWithNoLength.sql |   2 +
 .../testJoinFilterOfRowPreservedTable1.sql      |   2 +-
 .../testJoinFilterOfRowPreservedTable1.result   |   2 +-
 .../org/apache/tajo/plan/LogicalPlanner.java    |   4 +
 .../plan/expr/AggregationFunctionCallEval.java  |   4 +-
 .../apache/tajo/plan/expr/AlgebraicUtil.java    |   5 +
 .../org/apache/tajo/plan/expr/EvalNode.java     |  39 +-
 .../java/org/apache/tajo/plan/expr/InEval.java  |   2 +-
 .../plan/expr/PatternMatchPredicateEval.java    |   2 +-
 .../tajo/plan/expr/WindowFunctionEval.java      |   2 +-
 .../stream/TextFieldSerializerDeserializer.java |   8 +-
 .../storage/BinarySerializerDeserializer.java   |  10 +
 .../org/apache/tajo/storage/FrameTuple.java     |  14 +-
 .../org/apache/tajo/storage/MergeScanner.java   |   2 +-
 .../org/apache/tajo/storage/RowStoreUtil.java   |  20 +-
 .../org/apache/tajo/storage/StorageManager.java | 645 +------------------
 .../org/apache/tajo/storage/TableSpace.java     |  74 +++
 .../apache/tajo/storage/TableSpaceManager.java  | 254 ++++++++
 .../storage/TextSerializerDeserializer.java     |  10 +-
 .../tajo/storage/hbase/HBasePutAppender.java    |   4 +-
 .../apache/tajo/storage/hbase/HBaseScanner.java |   7 +-
 .../tajo/storage/hbase/HBaseStorageManager.java |  40 +-
 .../storage/hbase/TestHBaseStorageManager.java  |   5 +-
 .../org/apache/tajo/storage/FileAppender.java   |   2 +-
 .../apache/tajo/storage/FileStorageManager.java | 377 ++++++++++-
 .../storage/HashShuffleAppenderManager.java     |   4 +-
 .../tajo/storage/parquet/TajoWriteSupport.java  |   7 +
 .../text/TextFieldSerializerDeserializer.java   |   8 +-
 .../tajo/storage/TestCompressionStorages.java   |   5 +-
 .../tajo/storage/TestDelimitedTextFile.java     |   9 +-
 .../tajo/storage/TestFileStorageManager.java    |  11 +-
 .../apache/tajo/storage/TestFileSystems.java    |   3 +-
 .../org/apache/tajo/storage/TestLineReader.java |   9 +-
 .../apache/tajo/storage/TestMergeScanner.java   |   7 +-
 .../org/apache/tajo/storage/TestStorages.java   |  94 ++-
 .../apache/tajo/storage/index/TestBSTIndex.java |  53 +-
 .../index/TestSingleCSVFileBSTIndex.java        |   5 +-
 .../apache/tajo/storage/json/TestJsonSerDe.java |   8 +-
 123 files changed, 2389 insertions(+), 2900 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 42d9b36,21f5e5a..4372e04
--- a/CHANGES
+++ b/CHANGES
@@@ -123,8 -128,16 +128,18 @@@ Release 0.11.0 - unrelease
  
    BUG FIXES
  
 +    TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
 +
+     TAJO-1605: Fix master build failure on jdk 1.6. (jinho)
+ 
+     TAJO-1485: Datum 'Char' returned only 1byte.
+     (Contributed by DaeMyung Kang, Committed by jihoon)
+ 
+     TAJO-1586: TajoMaster HA startup failure on Yarn. (jaehwa)
+ 
+     TAJO-1598: TableMeta should change equals mechanism.
+     (Contributed by DaeMyung Kang, Committed by jihoon)
+ 
      TAJO-1593: Add missing stop condition to Taskrunner. (jinho)
  
      TAJO-1556: "insert into select" with reordered column list does not work.

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index 31dfb13,6df26b7..3341c68
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@@ -21,6 -21,6 +21,7 @@@ package org.apache.tajo.cli.tsql.comman
  import org.apache.commons.lang.CharUtils;
  import org.apache.commons.lang.StringEscapeUtils;
  import org.apache.tajo.TajoConstants;
++import org.apache.tajo.catalog.CatalogUtil;
  import org.apache.tajo.catalog.Column;
  import org.apache.tajo.catalog.TableDesc;
  import org.apache.tajo.catalog.partition.PartitionMethodDesc;

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 2dd7fa8,ff42d5f..04315eb
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@@ -46,11 -47,8 +46,12 @@@ import org.apache.tajo.ipc.TajoWorkerPr
  import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
  import org.apache.tajo.plan.LogicalPlan;
  import org.apache.tajo.plan.logical.*;
 +import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
  import org.apache.tajo.plan.util.PlannerUtil;
 -import org.apache.tajo.storage.*;
 +import org.apache.tajo.storage.FileStorageManager;
 +import org.apache.tajo.storage.StorageConstants;
 +import org.apache.tajo.storage.StorageManager;
++import org.apache.tajo.storage.TableSpaceManager;
  import org.apache.tajo.storage.fragment.FileFragment;
  import org.apache.tajo.storage.fragment.Fragment;
  import org.apache.tajo.storage.fragment.FragmentConvertor;

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 211d438,4612d45..7df48a4
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@@ -68,21 -54,10 +68,21 @@@ public class BSTIndexScanExec extends P
      super(context, scanNode.getInSchema(), scanNode.getOutSchema());
      this.scanNode = scanNode;
      this.qual = scanNode.getQual();
 -    this.datum = datum;
 +
 +    SortSpec[] keySortSpecs = new SortSpec[predicates.length];
 +    values = new Datum[predicates.length];
 +    for (int i = 0; i < predicates.length; i++) {
 +      keySortSpecs[i] = predicates[i].getKeySortSpec();
 +      values[i] = predicates[i].getValue();
 +    }
 +
 +    TupleComparator comparator = new BaseTupleComparator(keySchema,
 +        keySortSpecs);
 +
 +    Schema fileScanOutSchema = mergeSubSchemas(inSchema, keySchema, scanNode.getTargets(),
qual);
  
-     this.fileScanner = StorageManager.getSeekableScanner(context.getConf(),
+     this.fileScanner = TableSpaceManager.getSeekableScanner(context.getConf(),
 -        scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema);
 +        scanNode.getTableDesc().getMeta(), inSchema, fragment, fileScanOutSchema);
      this.fileScanner.init();
      this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets());
  
@@@ -92,23 -67,6 +92,23 @@@
      this.reader.open();
    }
  
 +  private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[]
targets, EvalNode qual) {
 +    Schema mergedSchema = new Schema();
 +    Set<Column> qualAndTargets = TUtil.newHashSet();
 +    qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual));
 +    for (Target target : targets) {
 +      qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()));
 +    }
-     for (Column column : originalSchema.getColumns()) {
++    for (Column column : originalSchema.getRootColumns()) {
 +      if (subSchema.contains(column)
 +          || qualAndTargets.contains(column)
 +          || qualAndTargets.contains(column)) {
 +        mergedSchema.addColumn(column);
 +      }
 +    }
 +    return mergedSchema;
 +  }
 +
    @Override
    public void init() throws IOException {
      super.init();

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 991cb25,aeb4166..fc9212c
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@@ -55,8 -54,8 +54,9 @@@ import org.apache.tajo.plan.verifier.Pr
  import org.apache.tajo.plan.verifier.VerificationState;
  import org.apache.tajo.plan.verifier.VerifyException;
  import org.apache.tajo.storage.StorageManager;
+ import org.apache.tajo.storage.TableSpaceManager;
  import org.apache.tajo.util.CommonTestingUtil;
 +import org.apache.tajo.util.IPCUtil;
  
  import java.io.IOException;
  import java.sql.SQLException;

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index 12f4b79,d77001c..640ec60
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@@ -36,14 -36,7 +36,13 @@@ import org.apache.tajo.algebra.Expr
  import org.apache.tajo.algebra.JsonHelper;
  import org.apache.tajo.catalog.CatalogService;
  import org.apache.tajo.catalog.TableDesc;
- import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
  import org.apache.tajo.conf.TajoConf;
 +import org.apache.tajo.plan.LogicalOptimizer;
 +import org.apache.tajo.plan.LogicalPlan;
 +import org.apache.tajo.plan.LogicalPlanner;
 +import org.apache.tajo.plan.logical.LogicalRootNode;
 +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
 +import org.apache.tajo.plan.util.PlannerUtil;
  import org.apache.tajo.engine.planner.global.MasterPlan;
  import org.apache.tajo.engine.query.QueryContext;
  import org.apache.tajo.exception.UnimplementedException;

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index ab94aac,5efdede..35cd612
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@@ -80,9 -80,9 +80,9 @@@ public class TestExecutionBlockCursor 
  
      analyzer = new SQLAnalyzer();
      logicalPlanner = new LogicalPlanner(catalog);
 -    optimizer = new LogicalOptimizer(conf);
 +    optimizer = new LogicalOptimizer(conf, catalog);
  
-     StorageManager sm  = StorageManager.getFileStorageManager(conf);
+     StorageManager sm  = TableSpaceManager.getFileStorageManager(conf);
      dispatcher = new AsyncDispatcher();
      dispatcher.init(conf);
      dispatcher.start();

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index ed6a570,c6b7354..7517e3a
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@@ -18,12 -18,10 +18,13 @@@
  
  package org.apache.tajo.plan.expr;
  
 +import org.apache.tajo.algebra.*;
  import org.apache.tajo.catalog.Column;
 +import org.apache.tajo.plan.PlanningException;
 +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
  
  import java.util.ArrayList;
+ import java.util.Collection;
  import java.util.List;
  import java.util.Map;
  import java.util.Stack;


Mime
View raw message