tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [7/7] tajo git commit: TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
Date Mon, 11 May 2015 22:30:33 GMT
TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)

Closes #564


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/53ed1c37
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/53ed1c37
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/53ed1c37

Branch: refs/heads/master
Commit: 53ed1c376f4969b5a2c54e30ddbd8113847a8768
Parents: 70df3a8 0d1bf41
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Mon May 11 15:29:17 2015 -0700
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Mon May 11 15:29:17 2015 -0700

----------------------------------------------------------------------
 CHANGES                                         |  11 +-
 .../org/apache/tajo/catalog/CatalogUtil.java    |  12 +-
 .../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
 .../java/org/apache/tajo/catalog/Schema.java    | 109 +++++-
 .../org/apache/tajo/catalog/SchemaUtil.java     |  33 +-
 .../tajo/catalog/store/HiveCatalogStore.java    |   4 +-
 .../cli/tsql/commands/DescTableCommand.java     |   4 +-
 .../org/apache/tajo/storage/RowStoreUtil.java   |   2 +-
 .../java/org/apache/tajo/storage/Tuple.java     |   2 +-
 .../java/org/apache/tajo/util/StringUtils.java  |  70 ++++
 .../main/java/org/apache/tajo/util/TUtil.java   |  32 --
 .../org/apache/tajo/engine/parser/SQLParser.g4  |   2 +-
 .../apache/tajo/engine/parser/SQLAnalyzer.java  |  17 +-
 .../engine/planner/PhysicalPlannerImpl.java     |  17 +-
 .../tajo/engine/planner/global/DataChannel.java |   4 +-
 .../engine/planner/global/GlobalPlanner.java    |   6 +-
 .../global/builder/DistinctGroupbyBuilder.java  |   6 +-
 .../DistinctGroupbyThirdAggregationExec.java    |   2 +-
 .../planner/physical/PhysicalPlanUtil.java      |   2 +-
 .../engine/planner/physical/SeqScanExec.java    |  27 +-
 .../tajo/engine/utils/TupleCacheScanner.java    |   2 +-
 .../org/apache/tajo/engine/utils/TupleUtil.java |   8 +-
 .../exec/NonForwardQueryResultFileScanner.java  |   2 +-
 .../NonForwardQueryResultSystemScanner.java     |  20 +-
 .../tajo/worker/ExecutionBlockContext.java      |  15 +-
 .../java/org/apache/tajo/worker/TaskRunner.java |   1 +
 .../resources/webapps/admin/catalogview.jsp     |   5 +-
 .../engine/function/TestFunctionLoader.java     |   5 +-
 .../tajo/engine/planner/TestLogicalPlanner.java |   8 +-
 .../apache/tajo/engine/query/TestCTASQuery.java |   4 +-
 .../tajo/engine/query/TestJoinBroadcast.java    |   2 +-
 .../engine/query/TestSelectNestedRecord.java    |  71 ++++
 .../java/org/apache/tajo/jdbc/TestTajoJdbc.java |   2 +-
 .../rs/resources/TestQueryResultResource.java   |   6 +-
 .../TestSelectNestedRecord/sample1/table.json   |   3 +
 .../TestSelectNestedRecord/tweets/sample1.json  |   4 +
 .../TestSelectNestedRecord/sample1_ddl.sql      |   7 +
 .../TestSelectNestedRecord/sample2_ddl.sql      |  19 ++
 .../testNestedFieldAsGroupbyKey1.sql            |   7 +
 .../testNestedFieldAsJoinKey1.sql               |   7 +
 .../TestSelectNestedRecord/testSelect1.sql      |   1 +
 .../TestSelectNestedRecord/testSelect2.sql      |  61 ++++
 .../TestSelectNestedRecord/tweets_ddl.sql       |  74 +++++
 .../testSelectWithParentheses2.sql              |   2 +-
 .../testNestedFieldAsGroupbyKey1.result         |   6 +
 .../testNestedFieldAsJoinKey1.result            |   6 +
 .../TestSelectNestedRecord/testSelect1.result   |   5 +
 .../TestSelectNestedRecord/testSelect2.result   |   6 +
 .../apache/tajo/jdbc/TajoDatabaseMetaData.java  |   2 +-
 .../tajo/plan/LogicalPlanPreprocessor.java      |   4 +-
 .../org/apache/tajo/plan/LogicalPlanner.java    |  32 +-
 .../apache/tajo/plan/expr/RowConstantEval.java  |   3 +-
 .../tajo/plan/expr/WindowFunctionEval.java      |   3 +-
 .../GreedyHeuristicJoinOrderAlgorithm.java      |   8 +-
 .../apache/tajo/plan/joinorder/JoinEdge.java    |   4 +-
 .../apache/tajo/plan/joinorder/JoinGraph.java   |   6 +-
 .../tajo/plan/logical/DistinctGroupbyNode.java  |   4 +-
 .../apache/tajo/plan/logical/EvalExprNode.java  |   3 +-
 .../apache/tajo/plan/logical/GroupbyNode.java   |  10 +-
 .../tajo/plan/logical/ProjectionNode.java       |   3 +-
 .../apache/tajo/plan/logical/RelationNode.java  |  12 +
 .../tajo/plan/logical/ShuffleFileWriteNode.java |   3 +-
 .../tajo/plan/logical/TruncateTableNode.java    |   4 +-
 .../apache/tajo/plan/logical/WindowAggNode.java |   7 +-
 .../tajo/plan/nameresolver/NameResolver.java    | 194 ++++++++---
 .../plan/nameresolver/ResolverByLegacy.java     |   8 +-
 .../plan/rewrite/rules/FilterPushDownRule.java  |   4 +-
 .../rewrite/rules/PartitionedTableRewriter.java |   4 +-
 .../org/apache/tajo/plan/util/PlannerUtil.java  |   2 +-
 .../tajo/plan/verifier/LogicalPlanVerifier.java |  24 +-
 .../plan/verifier/PreLogicalPlanVerifier.java   |   6 +-
 .../org/apache/tajo/rpc/AsyncRpcServer.java     |   6 +-
 .../org/apache/tajo/rpc/BlockingRpcServer.java  |   6 +-
 .../org/apache/tajo/storage/MergeScanner.java   |   2 +-
 .../org/apache/tajo/storage/RowStoreUtil.java   |   2 +-
 .../java/org/apache/tajo/storage/Scanner.java   |   9 +
 .../org/apache/tajo/storage/StorageManager.java |   4 +-
 .../storage/hbase/AbstractHBaseAppender.java    |   2 -
 .../tajo/storage/hbase/ColumnMapping.java       |   2 +-
 .../HBaseBinarySerializerDeserializer.java      |  10 +-
 .../apache/tajo/storage/hbase/HBaseScanner.java |  22 +-
 .../tajo/storage/hbase/HBaseStorageManager.java |   6 +-
 .../java/org/apache/tajo/storage/CSVFile.java   |   2 +-
 .../org/apache/tajo/storage/FileScanner.java    |   2 +-
 .../apache/tajo/storage/FileStorageManager.java |   2 +-
 .../apache/tajo/storage/avro/AvroScanner.java   |  25 +-
 .../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
 .../apache/tajo/storage/json/JsonLineSerDe.java |   5 +-
 .../tajo/storage/json/JsonLineSerializer.java   |   2 -
 .../storage/parquet/TajoRecordConverter.java    |   7 +-
 .../org/apache/tajo/storage/rcfile/RCFile.java  |  10 +-
 .../sequencefile/SequenceFileScanner.java       |  16 +-
 .../tajo/storage/text/CSVLineDeserializer.java  |  26 +-
 .../apache/tajo/storage/text/CSVLineSerDe.java  |   5 +-
 .../tajo/storage/text/DelimitedTextFile.java    |  11 +-
 .../tajo/storage/text/TextLineDeserializer.java |   4 +-
 .../apache/tajo/storage/text/TextLineSerDe.java |   3 +-
 .../apache/tajo/storage/TestMergeScanner.java   |  16 +-
 .../org/apache/tajo/storage/TestStorages.java   |  36 +-
 99 files changed, 1292 insertions(+), 462 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 9788307,84be66a..73fd2f3
--- a/CHANGES
+++ b/CHANGES
@@@ -24,6 -24,6 +24,8 @@@ Release 0.11.0 - unrelease
  
    IMPROVEMENT
  
++    TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
++
      TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, 
      Committed by hyunsik)
  

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index e2cfe8f,322818d..2c856e1
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@@ -175,7 -175,19 +175,19 @@@ public class TestMergeScanner 
      Tuple tuple;
      while ((tuple = scanner.next()) != null) {
        totalCounts++;
-       if (isProjectableStorage(meta.getStoreType())) {
+ 
 -      if (storeType == StoreType.RAW) {
++      if (storeType.equalsIgnoreCase("RAW")) {
+         assertEquals(4, tuple.size());
+         assertNotNull(tuple.get(0));
+         assertNotNull(tuple.get(1));
+         assertNotNull(tuple.get(2));
+         assertNotNull(tuple.get(3));
+       } else if (scanner.isProjectable()) {
+         assertEquals(2, tuple.size());
+         assertNotNull(tuple.get(0));
+         assertNotNull(tuple.get(1));
+       } else {
+         assertEquals(4, tuple.size());
          assertNotNull(tuple.get(0));
          assertNull(tuple.get(1));
          assertNotNull(tuple.get(2));
@@@ -187,15 -199,15 +199,13 @@@
      assertEquals(tupleNum * 2, totalCounts);
  	}
  
 -  private static boolean isProjectableStorage(StoreType type) {
 -    switch (type) {
 -    case CSV:
 -    case SEQUENCEFILE:
 -    case RAW:
 -    case ROWFILE:
 -      return false;
 -    default:
 +  private static boolean isProjectableStorage(String type) {
 +    if (type.equalsIgnoreCase("RCFILE") ||
 +        type.equalsIgnoreCase("PARQUET") ||
-         type.equalsIgnoreCase("SEQUENCEFILE") ||
-         type.equalsIgnoreCase("CSV") ||
 +        type.equalsIgnoreCase("AVRO")) {
        return true;
 +    } else {
 +      return false;
      }
    }
  }

http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 7c75973,a735307..949e2e8
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@@ -313,9 -305,23 +305,23 @@@ public class TestStorages 
      assertEquals(tupleNum, tupleCnt);
    }
  
+   private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) {
+     if (projectable) {
+       assertTrue(tupleCnt + 2 == tuple.get(0).asInt8());
+       assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4());
+     } else {
+       // RAW and ROW always project all fields.
 -      if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) {
++      if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE"))
{
+         assertTrue(tuple.get(0) == null);
+       }
+       assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
+       assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+     }
+   }
+ 
    @Test
    public void testVariousTypes() throws IOException {
 -    boolean handleProtobuf = storeType != StoreType.JSON;
 +    boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
  
      Schema schema = new Schema();
      schema.addColumn("col1", Type.BOOLEAN);
@@@ -956,7 -962,7 +962,8 @@@
    @Test
    public void testLessThanSchemaSize() throws IOException {
      /* RAW is internal storage. It must be same with schema size */
-     if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){
 -    if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET)
{
++    if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")
++        || storeType.equalsIgnoreCase("PARQUET")){
        return;
      }
  


Mime
View raw message