tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject git commit: TAJO-705: CTAS always stores tables with CSV storage type into catalog. (jinho)
Date Tue, 25 Mar 2014 14:45:37 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 4209b83c2 -> 6a167aeaa


TAJO-705: CTAS always stores tables with CSV storage type into catalog. (jinho)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6a167aea
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6a167aea
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6a167aea

Branch: refs/heads/master
Commit: 6a167aeaaea4ed1735e808875037b9796713bc1f
Parents: 4209b83
Author: jinossy <jinossy@gmail.com>
Authored: Tue Mar 25 23:42:26 2014 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Tue Mar 25 23:42:26 2014 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../org/apache/tajo/catalog/CatalogUtil.java    | 14 +++++--
 .../tajo/engine/planner/LogicalPlanner.java     |  6 ++-
 .../apache/tajo/master/querymaster/Query.java   |  2 +-
 .../apache/tajo/engine/query/TestCTASQuery.java | 41 +++++++++++++++++---
 .../queries/TestCTASQuery/CtasWithOptions.sql   | 13 +++++++
 .../queries/TestCTASQuery/CtasWithStoreType.sql | 12 ++++++
 .../TestCTASQuery/testCtasWithOptions.sql       |  1 +
 .../TestCTASQuery/testCtasWithStoreType.sql     |  1 +
 .../TestCTASQuery/testCtasWithOptions.result    |  5 +++
 .../TestCTASQuery/testCtasWithStoreType.result  |  5 +++
 11 files changed, 92 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8c0254d..4609d08 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -283,6 +283,9 @@ Release 0.8.0 - unreleased
 
   BUG FIXES
 
+    TAJO-705: CTAS always stores tables with CSV storage type into catalog.
+    (jinho)
+
     TAJO-693: StatusUpdateTransition in QueryUnitAttempt handles TA_UPDATE 
     incorrectly. (hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 2ddbb17..5ad2eca 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -183,11 +183,9 @@ public class CatalogUtil {
       return StoreType.CSV;
     } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) {
       return StoreType.RAW;
-    } else if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) {
-      return StoreType.CSV;
     } else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) {
       return StoreType.ROWFILE;
-    }else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) {
+    } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) {
       return StoreType.RCFILE;
     } else if (typeStr.equalsIgnoreCase(StoreType.TREVNI.name())) {
       return StoreType.TREVNI;
@@ -195,6 +193,16 @@ public class CatalogUtil {
       return null;
     }
   }
+  public static Options newOptionsWithDefault(StoreType type) {
+    Options options = new Options();
+    if(StoreType.CSV == type){
+      options.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.CSVFILE_DELIMITER_DEFAULT);
+    } else if(StoreType.RCFILE == type){
+      options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.RCFILE_BINARY_SERDE);
+    }
+
+    return options;
+  }
 
   public static TableMeta newTableMeta(StoreType type) {
     return new TableMeta(type, new Options());

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 48e3c93..c479f1c 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1313,12 +1313,14 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
       createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
     }
 
+    // Set default options to be created.
+    Options options = CatalogUtil.newOptionsWithDefault(createTableNode.getStorageType());
     if (expr.hasParams()) {
-      Options options = new Options();
       options.putAll(expr.getParams());
-      createTableNode.setOptions(options);
     }
 
+    createTableNode.setOptions(options);
+
     if (expr.hasPartition()) {
       if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
         createTableNode.setPartitionMethod(getPartitionMethod(context, expr.getTableName(),
expr.getPartitionMethod()));

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 8467b4b..3a4df7b 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -482,10 +482,10 @@ public class Query implements EventHandler<QueryEvent> {
                           Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir)
throws Exception {
         CatalogService catalog = context.getWorkerContext().getCatalog();
         SubQuery lastStage = query.getSubQuery(finalExecBlockId);
-        TableMeta meta = lastStage.getTableMeta();
         TableStats stats = lastStage.getResultStats();
 
         CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan();
+        TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions());
 
         TableDesc tableDescTobeCreated =
             new TableDesc(

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index e61f509..82c4be9 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -19,15 +19,14 @@
 package org.apache.tajo.engine.query;
 
 import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.IntegrationTest;
 import org.apache.tajo.QueryTestCaseBase;
 import org.apache.tajo.TajoConstants;
 import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.CatalogService;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.*;
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.junit.Test;
@@ -38,8 +37,7 @@ import java.util.Map;
 
 import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
 import static org.apache.tajo.catalog.CatalogUtil.buildFQName;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 
 /**
@@ -178,4 +176,37 @@ public class TestCTASQuery extends QueryTestCaseBase {
     resultSetToString(res2);
     res2.close();
   }
+
+  @Test
+  public final void testCtasWithStoreType() throws Exception {
+    ResultSet res = executeFile("CtasWithStoreType.sql");
+    res.close();
+
+    ResultSet res2 = executeQuery();
+    resultSetToString(res2);
+    res2.close();
+
+    TableDesc desc =  client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
+    assertNotNull(desc);
+    assertEquals(CatalogProtos.StoreType.RCFILE, desc.getMeta().getStoreType());
+  }
+
+  @Test
+  public final void testCtasWithOptions() throws Exception {
+    ResultSet res = executeFile("CtasWithOptions.sql");
+    res.close();
+
+    ResultSet res2 = executeQuery();
+    resultSetToString(res2);
+    res2.close();
+
+    TableDesc desc =  client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
+    assertNotNull(desc);
+    assertEquals(CatalogProtos.StoreType.CSV, desc.getMeta().getStoreType());
+
+
+    Options options = desc.getMeta().getOptions();
+    assertNotNull(options);
+    assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(CatalogConstants.CSVFILE_DELIMITER));
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
new file mode 100644
index 0000000..a393b9d
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql
@@ -0,0 +1,13 @@
+create table testCtasWithOptions (col1 float, col2 float)
+using csv with ('csvfile.delimiter'='\u0001') as
+select
+  sum(l_orderkey) as total1,
+  avg(l_partkey) as total2
+from
+  lineitem
+group by
+  l_quantity
+order by
+  l_quantity
+limit
+  3;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
new file mode 100644
index 0000000..bee1a89
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql
@@ -0,0 +1,12 @@
+create table testCtasWithStoreType (col1 float, col2 float) using rcfile as
+select
+  sum(l_orderkey) as total1,
+  avg(l_partkey) as total2
+from
+  lineitem
+group by
+  l_quantity
+order by
+  l_quantity
+limit
+  3;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
new file mode 100644
index 0000000..ac25e10
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql
@@ -0,0 +1 @@
+select * from testCtasWithOptions;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
new file mode 100644
index 0000000..9e7e337
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql
@@ -0,0 +1 @@
+select * from testCtasWithStoreType;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
new file mode 100644
index 0000000..ea78854
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result
@@ -0,0 +1,5 @@
+col1,col2,key
+-------------------------------
+1.0,1.0,17.0
+1.0,1.0,36.0
+2.0,2.0,38.0
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/6a167aea/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
new file mode 100644
index 0000000..ea78854
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result
@@ -0,0 +1,5 @@
+col1,col2,key
+-------------------------------
+1.0,1.0,17.0
+1.0,1.0,36.0
+2.0,2.0,38.0
\ No newline at end of file


Mime
View raw message