tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject git commit: TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik)
Date Tue, 27 May 2014 18:33:08 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 31e247cd6 -> a191b1dc7


TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik)

Closes #21


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a191b1dc
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a191b1dc
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a191b1dc

Branch: refs/heads/master
Commit: a191b1dc7ad3189c825069084fc6fc6167a4eb87
Parents: 31e247c
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Wed May 28 03:01:17 2014 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Wed May 28 03:01:17 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |   2 +
 .../org/apache/tajo/algebra/CreateTable.java    |  11 ++
 .../org/apache/tajo/engine/parser/SQLParser.g4  |   3 +-
 .../tajo/engine/parser/HiveQLAnalyzer.java      |   7 +-
 .../apache/tajo/engine/parser/SQLAnalyzer.java  |   6 +-
 .../tajo/engine/planner/LogicalPlanner.java     |  31 +++-
 .../tajo/engine/parser/TestHiveQLAnalyzer.java  |  13 ++
 .../tajo/engine/parser/TestSQLAnalyzer.java     |   9 +
 .../tajo/engine/query/TestCreateTable.java      | 176 +++++++++++++++++++
 .../queries/default/create_table_like_1.sql     |   1 +
 10 files changed, 255 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8107f81..66b5641 100644
--- a/CHANGES
+++ b/CHANGES
@@ -15,6 +15,8 @@ Release 0.9.0 - unreleased
 
   IMPROVEMENT
 
+    TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik)
+
     TAJO-825: Datetime type refactoring. (Hyoungjun Kim via jihoon)
 
     TAJO-811: add simple fifo scheduler support. (jinho)

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index f60b571..c74677d 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@ -47,6 +47,8 @@ public class CreateTable extends Expr {
   private PartitionMethodDescExpr partition;
   @Expose @SerializedName("IfNotExists")
   private boolean ifNotExists;
+  @Expose @SerializedName("LikeParentTable")
+  private String likeParentTable;
 
   public CreateTable(final String tableName, boolean ifNotExists) {
     super(OpType.CreateTable);
@@ -147,6 +149,15 @@ public class CreateTable extends Expr {
     return ifNotExists;
   }
 
+  public void setLikeParentTable(String parentTable)  {
+    this.likeParentTable = parentTable;
+  }
+
+  public String getLikeParentTableName()  {
+    return likeParentTable;
+  }
+
+
   @Override
   public int hashCode() {
     return Objects.hashCode(

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 0076794..9570457 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -94,6 +94,7 @@ create_table_statement
     (param_clause)? (table_partitioning_clauses)? (AS query_expression)?
   | CREATE TABLE (if_not_exists)? table_name (USING file_type=identifier)?
     (param_clause)? (table_partitioning_clauses)? AS query_expression
+  | CREATE TABLE (if_not_exists)? table_name LIKE like_table_name=table_name
   ;
 
 table_elements
@@ -804,7 +805,7 @@ boolean_primary
   ;
 
 boolean_predicand
-  : parenthesized_boolean_value_expression 
+  : parenthesized_boolean_value_expression
   | nonparenthesized_value_expression_primary
   ;
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
index de4b159..2cec22b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
@@ -1425,6 +1425,11 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr>
{
         createTable.setExternal();
       }
 
+      if(ctx.KW_LIKE() != null)  {
+        createTable.setLikeParentTable(ctx.likeName.getText());
+        return createTable;
+      }
+
       if (ctx.tableFileFormat() != null) {
         if (ctx.tableFileFormat().KW_RCFILE() != null) {
           createTable.setStorageType("rcfile");
@@ -1548,4 +1553,4 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr>
{
       return Character.toUpperCase((char) returnChar);
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index e7de8f6..fe65d47 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1006,8 +1006,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
 
   @Override
   public Expr visitCreate_table_statement(SQLParser.Create_table_statementContext ctx) {
-    String tableName = ctx.table_name().getText();
+    String tableName = ctx.table_name(0).getText();
     CreateTable createTable = new CreateTable(tableName, checkIfExist(ctx.if_not_exists()));
+    if(checkIfExist(ctx.LIKE()))  {
+      createTable.setLikeParentTable(ctx.like_table_name.getText());
+      return createTable;
+    }
 
     if (checkIfExist(ctx.EXTERNAL())) {
       createTable.setExternal();

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 0780d4f..76dae24 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1315,6 +1315,33 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
     return dropDatabaseNode;
   }
 
+  public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, CreateTableNode
createTableNode)
+    throws PlanningException {
+    String parentTableName = expr.getLikeParentTableName();
+
+    if (CatalogUtil.isFQTableName(parentTableName) == false) {
+      parentTableName =
+	CatalogUtil.buildFQName(context.session.getCurrentDatabase(),
+				parentTableName);
+    }
+    TableDesc parentTableDesc = catalog.getTableDesc(parentTableName);
+    if(parentTableDesc == null)
+      throw new PlanningException("Table '"+parentTableName+"' does not exist");
+    PartitionMethodDesc partitionDesc = parentTableDesc.getPartitionMethod();
+    createTableNode.setTableSchema(parentTableDesc.getSchema());
+    createTableNode.setPartitionMethod(partitionDesc);
+
+    createTableNode.setStorageType(parentTableDesc.getMeta().getStoreType());
+    createTableNode.setOptions(parentTableDesc.getMeta().getOptions());
+
+    createTableNode.setExternal(parentTableDesc.isExternal());
+    if(parentTableDesc.isExternal()) {
+      createTableNode.setPath(parentTableDesc.getPath());
+    }
+    return createTableNode;
+  }
+
+
   @Override
   public LogicalNode visitCreateTable(PlanContext context, Stack<Expr> stack, CreateTable
expr)
       throws PlanningException {
@@ -1329,7 +1356,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
       createTableNode.setTableName(
           CatalogUtil.buildFQName(context.session.getCurrentDatabase(), expr.getTableName()));
     }
-
+    // This is CREATE TABLE <tablename> LIKE <parentTable>
+    if(expr.getLikeParentTableName() != null)
+      return handleCreateTableLike(context, expr, createTableNode);
 
     if (expr.hasStorageType()) { // If storage type (using clause) is specified
       createTableNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType()));

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
index ef21dc3..45b5f44 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
@@ -19,12 +19,15 @@
 package org.apache.tajo.engine.parser;
 
 import com.google.common.base.Preconditions;
+
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.tajo.engine.parser.SQLParser.SqlContext;
+import org.apache.tajo.algebra.CreateTable;
 import org.apache.tajo.algebra.Expr;
+import org.apache.tajo.algebra.OpType;
 import org.apache.tajo.util.FileUtil;
 import org.junit.Test;
 
@@ -268,4 +271,14 @@ public class TestHiveQLAnalyzer {
   public void testDrop() throws IOException {
     compareJsonResult("drop_table.sql");
   }
+
+  @Test
+  public void testCreateTableLike1() throws IOException {
+    String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql"));
+    Expr expr = parseQuery(sql);
+    assertEquals(OpType.CreateTable, expr.getType());
+    CreateTable createTable = (CreateTable) expr;
+    assertEquals("orig_name", createTable.getLikeParentTableName());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
index 2010502..5423813 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
@@ -270,6 +270,15 @@ public class TestSQLAnalyzer {
   }
 
   @Test
+  public void testCreateTableLike1() throws IOException {
+    String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql"));
+    Expr expr = parseQuery(sql);
+    assertEquals(OpType.CreateTable, expr.getType());
+    CreateTable createTable = (CreateTable) expr;
+    assertEquals("orig_name", createTable.getLikeParentTableName());
+  }
+
+  @Test
   public void testCreateTablePartitionByHash1() throws IOException {
     String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_hash_1.sql"));
     Expr expr = parseQuery(sql);

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index 2d289ba..0655700 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -23,9 +23,14 @@ import org.apache.hadoop.fs.Path;
 import org.apache.tajo.IntegrationTest;
 import org.apache.tajo.QueryTestCaseBase;
 import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.util.KeyValueSet;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -34,6 +39,7 @@ import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 @Category(IntegrationTest.class)
 public class TestCreateTable extends QueryTestCaseBase {
@@ -357,4 +363,174 @@ public class TestCreateTable extends QueryTestCaseBase {
     createdNames = executeDDL("table1_ddl.sql", "table1", "varchar");
     assertTableExists(createdNames.get(0));
   }
+
+  private boolean isClonedSchema(Schema origSchema, Schema newSchema)  {
+    // Check schema of tables
+    boolean schemaEqual =
+      (origSchema.size() == newSchema.size());
+    if(schemaEqual == false)  {
+      fail("Number of columns in schema not equal");
+      return false;
+    }
+
+    for(int col = 0; col < origSchema.size(); col++)  {
+      Column colA = origSchema.getColumn(col);
+      Column colB = newSchema.getColumn(col);
+      if(colA.getSimpleName().equals(colB.getSimpleName()) == false)  {
+        fail("Column names at index " + col + " do not match");
+        return false;
+      }
+      if(colA.getDataType().equals(colB.getDataType()) == false) {
+        fail("Column datatypes at index " + col + " do not match");
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private boolean isClonedTable(String orignalTable, String newTable) throws Exception  {
+    assertTableExists(newTable);
+    TableDesc origTableDesc = client.getTableDesc(orignalTable);
+    TableDesc newTableDesc = client.getTableDesc(newTable);
+
+    if(isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) {
+      fail("Schema of input tables do not match");
+      return false;
+    }
+
+    // Check partition information
+    PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod();
+    PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod();
+    if(origPartMethod != null) {
+      if(newPartMethod == null)  {
+        fail("New table does not have partition info");
+        return false;
+      }
+      if(isClonedSchema(origPartMethod.getExpressionSchema(),
+                        newPartMethod.getExpressionSchema()) == false) {
+	fail("Partition columns of input tables do not match");
+        return false;
+      }
+
+      if(origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false)
 {
+        fail("Partition type of input tables do not match");
+        return false;
+      }
+    }
+
+    // Check external flag
+    if(origTableDesc.isExternal() != newTableDesc.isExternal()) {
+      fail("External table flag on input tables not equal");
+      return false;
+    }
+
+    if(origTableDesc.getMeta() != null) {
+      TableMeta origMeta = origTableDesc.getMeta();
+      TableMeta newMeta = newTableDesc.getMeta();
+      if(origMeta.getStoreType().equals(newMeta.getStoreType()) == false) {
+        fail("Store type of input tables not equal");
+        return false;
+      }
+
+      KeyValueSet origOptions = origMeta.getOptions();
+      KeyValueSet newOptions = newMeta.getOptions();
+      if(origOptions.equals(newOptions) == false)  {
+        fail("Meta options of input tables not equal");
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Test
+  public final void testCreateTableLike1() throws Exception {
+    // Basic create table with default database
+    executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
+    executeString("CREATE TABLE table2 LIKE table1");
+    String testMsg = "testCreateTableLike1: Basic create table with default db";
+    assertTrue(testMsg,isClonedTable("table1","table2"));
+    executeString("DROP TABLE table1");
+    executeString("DROP TABLE table2");
+
+    // Basic create table with database
+    executeString("CREATE DATABASE d1").close();
+    executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
+    executeString("CREATE TABLE d1.table2 LIKE d1.table1");
+    testMsg = "testCreateTableLike1: Basic create table with db test failed";
+    assertTrue(testMsg, isClonedTable("d1.table1","d1.table2"));
+    executeString("DROP TABLE d1.table1");
+    executeString("DROP TABLE d1.table2");
+
+    // Table with non-default store type
+    executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
+    executeString("CREATE TABLE table2 LIKE table1");
+    testMsg = "testCreateTableLike1: Table with non-default store type test failed";
+    assertTrue(testMsg, isClonedTable("table1","table2"));
+    executeString("DROP TABLE table1");
+    executeString("DROP TABLE table2");
+
+    // Table with non-default meta options
+    executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close();
+    executeString("CREATE TABLE table2 LIKE table1");
+    testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
+    assertTrue(testMsg, isClonedTable("table1","table2"));
+    executeString("DROP TABLE table1");
+    executeString("DROP TABLE table2");
+
+
+    // Table with partitions (default partition type)
+    executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int,
c4 float, c5 text);").close();
+    executeString("CREATE TABLE table2 LIKE table1");
+    testMsg = "testCreateTableLike1: Table with partitions test failed";
+    assertTrue(testMsg, isClonedTable("table1","table2"));
+    executeString("DROP TABLE table1");
+    executeString("DROP TABLE table2");
+
+
+    // Table with external flag
+    // Use existing file as input for creating external table
+    String className = getClass().getSimpleName();
+    Path currentDatasetPath = new Path(datasetBasePath, className);
+    Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
+    executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION
'" + filePath.toUri() + "'").close();
+    executeString("CREATE TABLE table2 LIKE table3");
+    testMsg = "testCreateTableLike1: Table with external table flag test failed";
+    assertTrue(testMsg, isClonedTable("table3","table2"));
+    executeString("DROP TABLE table3");
+    executeString("DROP TABLE table2");
+
+
+    // Table created using CTAS
+    executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
+    executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2,c3 from table3;").close();
+    executeString("CREATE TABLE table2 LIKE table4");
+    testMsg = "testCreateTableLike1: Table using CTAS test failed";
+    assertTrue(testMsg, isClonedTable("table4","table2"));
+    executeString("DROP TABLE table3");
+    executeString("DROP TABLE table4");
+    executeString("DROP TABLE table2");
+
+
+    /* Enable when view is supported
+    // View
+    executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
+    executeString("CREATE VIEW table4(c1,c2,c3) AS SELECT c1*c1, c2, c2,c3 from table3;").close();
+    executeString("CREATE TABLE table2 LIKE table4");
+    testMsg = "testCreateTableLike1: Table using VIEW test failed";
+    assertTrue(testMsg, isClonedTable("table4","table2"));
+    executeString("DROP TABLE table3");
+    executeString("DROP TABLE table4");
+    executeString("DROP TABLE table2");
+    */
+
+    /*  Enable when partition type other than column is supported
+    // Table with partitions (range partition)
+    executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY RANGE (c1) (  PARTITION
c1 VALUES LESS THAN (2),  PARTITION c1 VALUES LESS THAN (5),  PARTITION c1 VALUES LESS THAN
(MAXVALUE) );").close();
+    executeString("CREATE TABLE table2 LIKE table1");
+    testMsg = "testCreateTableLike1: Table using non-default partition type failed";
+    assertTrue(testMsg, isClonedTable("table1","table2"));
+    executeString("DROP TABLE table1");
+    executeString("DROP TABLE table2");
+    */
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/default/create_table_like_1.sql b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
new file mode 100644
index 0000000..73f2f46
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql
@@ -0,0 +1 @@
+create table new_table like orig_name;


Mime
View raw message