tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject tajo git commit: TAJO-1674: Validation of CTAS schema mismatch.
Date Wed, 09 Sep 2015 01:54:36 GMT
Repository: tajo
Updated Branches:
  refs/heads/master a460978d9 -> bdd8e7685


TAJO-1674: Validation of CTAS schema mismatch.

Closes #726


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/bdd8e768
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/bdd8e768
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/bdd8e768

Branch: refs/heads/master
Commit: bdd8e76855eb0a79f22c67832edc5ce535a1a938
Parents: a460978
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Wed Sep 9 10:51:25 2015 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Wed Sep 9 10:51:25 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../org/apache/tajo/catalog/CatalogUtil.java    | 15 +++++++--
 .../tajo/engine/query/TestTablePartitions.java  | 32 +++-----------------
 .../TestCreateTable/negative/type_mismatch.sql  |  2 ++
 .../tajo/plan/verifier/LogicalPlanVerifier.java | 16 +++++++++-
 .../tajo/plan/verifier/SyntaxErrorUtil.java     |  5 +--
 6 files changed, 40 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8303b72..52d1ca5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -252,6 +252,8 @@ Release 0.11.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1674: Validation of CTAS schema mismatch. (hyunsik)
+
     TAJO-1819: Cannot find existing tables when pgsql catalog starts up. 
     (jihoon)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index ac76387..a6487b4 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -873,7 +873,11 @@ public class CatalogUtil {
     return pair;
   }
 
-  /* It is the relationship graph of type conversions. */
+  /*
+   * It is the relationship graph of type conversions.
+   * It contains tuples, each of which (LHS type, RHS type, Result type).
+   */
+
   public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap();
 
   static {
@@ -929,9 +933,16 @@ public class CatalogUtil {
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.FLOAT8, Type.TEXT, Type.TEXT);
 
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TIMESTAMP, Type.TIMESTAMP);
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT);
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.VARCHAR, Type.TEXT);
+
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TIMESTAMP, Type.TIMESTAMP);
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TEXT, Type.TEXT);
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.VARCHAR, Type.TEXT);
+
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TIMESTAMP, Type.TIMESTAMP);
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TEXT, Type.TEXT);
-    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT);
+    TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.VARCHAR, Type.TEXT);
 
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIME, Type.TIME, Type.TIME);
     TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.DATE, Type.DATE);

http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 52e7b54..94e5e71 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -37,14 +37,11 @@ import org.apache.tajo.engine.planner.global.DataChannel;
 import org.apache.tajo.engine.planner.global.ExecutionBlock;
 import org.apache.tajo.engine.planner.global.MasterPlan;
 import org.apache.tajo.ipc.ClientProtos;
-import org.apache.tajo.jdbc.FetchResultSet;
-import org.apache.tajo.jdbc.TajoMemoryResultSet;
 import org.apache.tajo.plan.logical.NodeType;
 import org.apache.tajo.querymaster.QueryMasterTask;
 import org.apache.tajo.storage.StorageConstants;
 import org.apache.tajo.util.CommonTestingUtil;
 import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.worker.TajoWorker;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -211,7 +208,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
 
     TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
     verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"},
-      tableDesc.getStats().getNumRows());
+        tableDesc.getStats().getNumRows());
 
     executeString("DROP TABLE " + tableName + " PURGE").close();
   }
@@ -254,7 +251,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
     }
 
     verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName,
-      new String[]{"key"}, desc.getStats().getNumRows());
+        new String[]{"key"}, desc.getStats().getNumRows());
 
     executeString("DROP TABLE " + tableName + " PURGE").close();
     res.close();
@@ -674,7 +671,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
     }
 
     verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1"},
-      desc.getStats().getNumRows());
+        desc.getStats().getNumRows());
 
     executeString("DROP TABLE " + tableName + " PURGE").close();
   }
@@ -733,7 +730,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
     }
 
     verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1",
"col2"},
-      desc.getStats().getNumRows());
+        desc.getStats().getNumRows());
 
     executeString("DROP TABLE " + tableName + " PURGE").close();
   }
@@ -1045,7 +1042,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
 
     TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
     verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col2"},
-      desc.getStats().getNumRows());
+        desc.getStats().getNumRows());
 
     executeString("DROP TABLE " + tableName + " PURGE").close();
   }
@@ -1081,25 +1078,6 @@ public class TestTablePartitions extends QueryTestCaseBase {
     executeString("DROP TABLE " + tableName + " PURGE").close();
   }
 
-  private MasterPlan getQueryPlan(ResultSet res) {
-    QueryId queryId;
-    if (res instanceof TajoMemoryResultSet) {
-      queryId = ((TajoMemoryResultSet) res).getQueryId();
-    } else {
-      queryId = ((FetchResultSet) res).getQueryId();
-    }
-
-    for (TajoWorker eachWorker : testingCluster.getTajoWorkers()) {
-      QueryMasterTask queryMasterTask = eachWorker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId,
true);
-      if (queryMasterTask != null) {
-        return queryMasterTask.getQuery().getPlan();
-      }
-    }
-
-    fail("Can't find query from workers" + queryId);
-    return null;
-  }
-
   @Test
   public void testScatteredHashShuffle() throws Exception {
     testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.varname,
"2");

http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql
b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql
new file mode 100644
index 0000000..1bff25c
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql
@@ -0,0 +1,2 @@
+CREATE TABLE MISMATCH1 (n_name text, n_comment text, n_nationkey int8, n_regionkey int8)
AS SELECT * FROM default.nation;
+CREATE TABLE MISMATCH2 (n_name text, n_comment text) PARTITION BY COLUMN (n_nationkey int8,
n_regionkey int8) AS SELECT * FROM default.nation;

http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
index 0b0968e..c87ceb3 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
@@ -19,6 +19,7 @@
 package org.apache.tajo.plan.verifier;
 
 import com.google.common.base.Preconditions;
+import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.Column;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.common.TajoDataTypes.Type;
@@ -31,6 +32,7 @@ import org.apache.tajo.plan.Target;
 import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
+import org.apache.tajo.util.TUtil;
 
 import java.util.Stack;
 
@@ -241,7 +243,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
   private static void ensureDomains(VerificationState state, Schema targetTableScheme, Schema
schema)
       throws TajoException {
     for (int i = 0; i < schema.size(); i++) {
-      if (!schema.getColumn(i).getDataType().equals(targetTableScheme.getColumn(i).getDataType()))
{
+
+      // null can be used anywhere
+      if (schema.getColumn(i).getDataType().getType() == Type.NULL_TYPE) {
+        continue;
+      }
+
+      // checking castable between two data types
+      if (!TUtil.containsInNestedMap(CatalogUtil.OPERATION_CASTING_MAP,
+          schema.getColumn(i).getDataType().getType(), targetTableScheme.getColumn(i).getDataType().getType()))
{
         Column targetColumn = targetTableScheme.getColumn(i);
         Column insertColumn = schema.getColumn(i);
         state.addVerification(makeDataTypeMisMatch(insertColumn, targetColumn));
@@ -254,6 +264,10 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
                                       CreateTableNode node, Stack<LogicalNode> stack)
throws TajoException {
     super.visitCreateTable(context, plan, block, node, stack);
     // here, we don't need check table existence because this check is performed in PreLogicalPlanVerifier.
+
+    if (node.hasSubQuery()) {
+      ensureDomains(context.state, node.getLogicalSchema(), node.getChild(0).getOutSchema());
+    }
     return node;
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/bdd8e768/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java
index 3c11b8c..bf4a0d6 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java
@@ -21,6 +21,7 @@ package org.apache.tajo.plan.verifier;
 import org.apache.tajo.catalog.Column;
 import org.apache.tajo.common.TajoDataTypes.DataType;
 import org.apache.tajo.error.Errors.ResultCode;
+import org.apache.tajo.exception.DataTypeMismatchException;
 import org.apache.tajo.plan.logical.NodeType;
 
 public class SyntaxErrorUtil {
@@ -29,8 +30,8 @@ public class SyntaxErrorUtil {
     return new SyntaxErrorException(ResultCode.SYNTAX_ERROR, message);
   }
 
-  public static SyntaxErrorException makeDataTypeMisMatch(Column src, Column target) {
-    return new SyntaxErrorException(ResultCode.DATATYPE_MISMATCH,
+  public static DataTypeMismatchException makeDataTypeMisMatch(Column src, Column target)
{
+    return new DataTypeMismatchException(
         src.getSimpleName(), src.getDataType().getType().name(),
         target.getSimpleName(), target.getDataType().getType().name());
   }


Mime
View raw message