tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From blrun...@apache.org
Subject tajo git commit: TAJO-1739: Add a statement for adding partition to TajoDump.
Date Fri, 11 Sep 2015 03:12:11 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.11.0 785352f09 -> 597df1db4


TAJO-1739: Add a statement for adding partition to TajoDump.


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/597df1db
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/597df1db
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/597df1db

Branch: refs/heads/branch-0.11.0
Commit: 597df1db4598d7de32fcb943ec0c14cf53670930
Parents: 785352f
Author: JaeHwa Jung <blrunner@apache.org>
Authored: Fri Sep 11 12:11:30 2015 +0900
Committer: JaeHwa Jung <blrunner@apache.org>
Committed: Fri Sep 11 12:11:30 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 +
 .../org/apache/tajo/catalog/DDLBuilder.java     | 43 ++++++++++++++++-
 .../src/main/proto/CatalogProtos.proto          |  5 ++
 .../org/apache/tajo/cli/tools/TajoDump.java     | 14 ++++++
 .../apache/tajo/client/CatalogAdminClient.java  | 10 ++++
 .../tajo/client/CatalogAdminClientImpl.java     | 22 +++++++++
 .../org/apache/tajo/client/TajoClientImpl.java  |  6 +++
 .../main/proto/TajoMasterClientProtocol.proto   |  1 +
 .../org/apache/tajo/cli/tools/TestTajoDump.java | 51 ++++++++++++++++++--
 .../TestTajoDump/testPartitionsDump.result      | 33 +++++++++++++
 .../tajo/master/TajoMasterClientService.java    | 32 +++++++++++-
 11 files changed, 213 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2e6dcb3..4b8101c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -528,6 +528,8 @@ Release 0.11.0 - unreleased
   
   TASKS
 
+    TAJO-1739: Add a statement for adding partition to TajoDump. (jaehwa)
+
     TAJO-1833: Refine LogicalPlanPreprocessor to add new rules easily.
     (jihoon)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 2923654..920fe83 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -19,8 +19,10 @@
 package org.apache.tajo.catalog;
 
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
 import org.apache.tajo.util.KeyValueSet;
 
+import java.util.List;
 import java.util.Map;
 
 public class DDLBuilder {
@@ -83,7 +85,7 @@ public class DDLBuilder {
       sb.append(sortSpec.isAscending() ? "asc" : "desc").append(" ");
       sb.append(sortSpec.isNullFirst() ? "null first" : "null last").append(", ");
     }
-    sb.replace(sb.length()-2, sb.length()-1, " )");
+    sb.replace(sb.length() - 2, sb.length() - 1, " )");
 
     sb.append(" location '").append(desc.getIndexPath()).append("';");
 
@@ -148,4 +150,43 @@ public class DDLBuilder {
     }
     sb.append(")");
   }
+
+  /**
+   * Build alter table add partition statement
+   *
+   * @param table TableDesc to be build
+   * @param partition PartitionDescProto to be build
+   * @return
+   */
+  public static String buildDDLForAddPartition(TableDesc table, PartitionDescProto partition)
{
+    StringBuilder sb = new StringBuilder();
+    sb.append("ALTER TABLE ").append(CatalogUtil.denormalizeIdentifier(table.getName()))
+      .append(" ADD IF NOT EXISTS PARTITION (");
+
+
+    List<Column> colums = table.getPartitionMethod().getExpressionSchema().getAllColumns();
+
+    String[] splitPartitionName = partition.getPartitionName().split("/");
+
+    for(int i = 0; i < splitPartitionName.length; i++) {
+      String[] partitionColumnValue = splitPartitionName[i].split("=");
+      if (i > 0) {
+        sb.append(",");
+      }
+
+      switch (colums.get(i).getDataType().getType()) {
+        case TEXT:
+        case TIME:
+        case TIMESTAMP:
+        case DATE:
+          sb.append(partitionColumnValue[0]).append("='").append(partitionColumnValue[1]).append("'");
+          break;
+        default:
+          sb.append(partitionColumnValue[0]).append("=").append(partitionColumnValue[1]);
+          break;
+      }
+    }
+    sb.append(") LOCATION '").append(partition.getPath()).append("';\n");
+    return sb.toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 04b5345..cfac82f 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -407,3 +407,8 @@ message IndexResponse {
   required ReturnState state = 1;
   optional IndexDescProto indexDesc = 2;
 }
+
+message PartitionListResponse {
+  required ReturnState state = 1;
+  repeated PartitionDescProto partition = 2;
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
index 12515a8..a50ce7b 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
@@ -22,7 +22,9 @@ import com.google.protobuf.ServiceException;
 import org.apache.commons.cli.*;
 import org.apache.tajo.auth.UserRoleInfo;
 import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.partition.PartitionDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
 import org.apache.tajo.client.TajoClient;
 import org.apache.tajo.client.TajoClientImpl;
 import org.apache.tajo.conf.TajoConf;
@@ -185,6 +187,18 @@ public class TajoDump {
           writer.write(DDLBuilder.buildDDLForBaseTable(table));
         }
 
+        if (table.hasPartition()) {
+          writer.write("\n\n");
+          writer.write("--\n");
+          writer.write(String.format("-- Table Partitions: %s%n", tableName));
+          writer.write("--\n");
+          List<PartitionDescProto> partitionProtos = client.getAllPartitions(tableName);
+          for (PartitionDescProto eachPartitionProto : partitionProtos) {
+            writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto));
+          }
+          writer.write("\n\n");
+        }
+
         if (client.hasIndexes(tableName)) {
           List<CatalogProtos.IndexDescProto> indexeProtos = client.getIndexes(tableName);
           for (CatalogProtos.IndexDescProto eachIndexProto : indexeProtos) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
index 7acc9b9..a5196dc 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
@@ -25,6 +25,7 @@ import org.apache.tajo.catalog.TableMeta;
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
 import org.apache.tajo.exception.*;
 
 import java.io.Closeable;
@@ -136,6 +137,15 @@ public interface CatalogAdminClient extends Closeable {
    */
   TableDesc getTableDesc(final String tableName) throws UndefinedTableException;
 
+  /**
+   * Get all partition lists of specified table.
+   *
+   * @param tableName The table name to get. This name is case sensitive.
+   * @return lists of partitions
+   */
+  List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
+    UndefinedTableException, UndefinedPartitionMethodException;
+
   List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName);
 
   IndexDescProto getIndex(final String indexName);

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index af1278d..fb13f07 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -241,6 +241,28 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
   }
 
   @Override
+  public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
+    UndefinedTableException, UndefinedPartitionMethodException {
+
+    final BlockingInterface stub = conn.getTMStub();
+
+    PartitionListResponse response;
+    try {
+      response = stub.getPartitionsByTableName(null,
+        conn.getSessionedString(tableName));
+    } catch (ServiceException e) {
+      throw new RuntimeException(e);
+    }
+
+    throwsIfThisError(response.getState(), UndefinedDatabaseException.class);
+    throwsIfThisError(response.getState(), UndefinedTableException.class);
+    throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class);
+
+    ensureOk(response.getState());
+    return response.getPartitionList();
+  }
+
+  @Override
   public List<FunctionDescProto> getFunctions(final String functionName) {
 
     final BlockingInterface stub = conn.getTMStub();

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
index 5a689a1..3d69309 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
@@ -29,6 +29,7 @@ import org.apache.tajo.catalog.TableMeta;
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
 import org.apache.tajo.exception.*;
 import org.apache.tajo.ipc.ClientProtos.*;
 import org.apache.tajo.jdbc.TajoMemoryResultSet;
@@ -235,6 +236,11 @@ public class TajoClientImpl extends SessionConnection implements TajoClient,
Que
     return catalogClient.getFunctions(functionName);
   }
 
+  public List<PartitionDescProto> getAllPartitions(final String tableName) throws UndefinedDatabaseException,
+    UndefinedTableException, UndefinedPartitionMethodException {
+    return catalogClient.getAllPartitions(tableName);
+  }
+
   @Override
   public IndexDescProto getIndex(String indexName) {
     return catalogClient.getIndex(indexName);

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
index 78f0f30..fd76c80 100644
--- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
+++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
@@ -70,6 +70,7 @@ service TajoMasterClientProtocolService {
   rpc getTableList(SessionedStringProto) returns (StringListResponse);
   rpc getTableDesc(SessionedStringProto) returns (TableResponse);
   rpc getFunctionList(SessionedStringProto) returns (FunctionListResponse);
+  rpc getPartitionsByTableName(SessionedStringProto) returns (PartitionListResponse);
 
   // Index Management APIs
   rpc getIndexWithName(SessionedStringProto) returns (IndexResponse);

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
index aa8070e..365aca6 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
@@ -79,13 +79,13 @@ public class TestTajoDump extends QueryTestCaseBase {
   }
 
   @Test
-  public void testDump3() throws Exception {
+  public void testDump4() throws Exception {
     if (!testingCluster.isHiveCatalogStoreRunning()) {
       executeString("CREATE TABLE \"" + getCurrentDatabase() +
-          "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)");
+        "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)");
 
       executeString("CREATE INDEX test_idx on \"" + getCurrentDatabase()
-          + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )");
+        + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )");
 
       try {
         UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
@@ -94,8 +94,9 @@ public class TestTajoDump extends QueryTestCaseBase {
         TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
         printWriter.flush();
         printWriter.close();
+
         assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"},
-            new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(),
"test_idx").toString()});
+          new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()});
         bos.close();
       } finally {
         executeString("DROP INDEX test_idx");
@@ -104,6 +105,48 @@ public class TestTajoDump extends QueryTestCaseBase {
     }
   }
 
+  @Test
+  public void testPartitionsDump() throws Exception {
+    if (!testingCluster.isHiveCatalogStoreRunning()) {
+      executeString("create table \"" + getCurrentDatabase() + "\".\"TableName3\""
+          + " (\"col1\" int4, \"col2\" int4) "
+          + " partition by column(\"col3\" int4, \"col4\" int4)"
+      );
+
+      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" +
+        " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)");
+
+      executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\""
+          + " (\"col1\" int4, \"col2\" int4) "
+          + " partition by column(\"col3\" TEXT, \"col4\" date)"
+      );
+
+      executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" +
+        " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')");
+
+      try {
+        UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        PrintWriter printWriter = new PrintWriter(bos);
+        TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
+        printWriter.flush();
+        printWriter.close();
+
+        String[] paramValues = new String[] {
+          TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName3").toString()
+          , TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName4").toString()
+        };
+
+        assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray())
+          , new String[]{"${partition.path1}", "${partition.path2}"}, paramValues);
+        bos.close();
+      } finally {
+        executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"");
+        executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"");
+      }
+    }
+  }
+
   private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys,
String[] paramValues)
       throws Exception {
     FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
new file mode 100644
index 0000000..696bb19
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
@@ -0,0 +1,33 @@
+--
+-- Tajo database dump
+--
+
+
+--
+-- Database name: "TestTajoDump"
+--
+
+CREATE DATABASE IF NOT EXISTS "TestTajoDump";
+
+--
+-- Name: "TestTajoDump"."TableName3"; Type: TABLE; Storage: TEXT
+--
+CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|')
PARTITION BY COLUMN(col3 INT4, col4 INT4);
+
+--
+-- Table Partitions: TableName3
+--
+ALTER TABLE "TestTajoDump"."TableName3" ADD IF NOT EXISTS PARTITION (col3=1,col4=2) LOCATION
'${partition.path1}/col3=1/col4=2';
+
+
+
+
+--
+-- Name: "TestTajoDump"."TableName4"; Type: TABLE; Storage: TEXT
+--
+CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|')
PARTITION BY COLUMN(col3 TEXT, col4 DATE);
+
+--
+-- Table Partitions: TableName4
+--
+ALTER TABLE "TestTajoDump"."TableName4" ADD IF NOT EXISTS PARTITION (col3='tajo',col4='2015-09-01')
LOCATION '${partition.path2}/col3=tajo/col4=2015-09-01';
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/597df1db/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index d6ace8f..40bca04 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -915,7 +915,7 @@ public class TajoMasterClientService extends AbstractService {
         QueryContext queryContext = new QueryContext(conf, session);
 
         context.getGlobalEngine().getDDLExecutor().dropTable(queryContext, dropTable.getName(),
false,
-            dropTable.getPurge());
+          dropTable.getPurge());
         return OK;
 
       } catch (Throwable t) {
@@ -960,6 +960,36 @@ public class TajoMasterClientService extends AbstractService {
     }
 
     @Override
+    public PartitionListResponse getPartitionsByTableName(RpcController controller, SessionedStringProto
request)
+      throws ServiceException {
+
+      try {
+        Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+        String databaseName;
+        String tableName;
+        if (CatalogUtil.isFQTableName(request.getValue())) {
+          String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+          databaseName = splitted[0];
+          tableName = splitted[1];
+        } else {
+          databaseName = session.getCurrentDatabase();
+          tableName = request.getValue();
+        }
+
+        List<PartitionDescProto> partitions = catalog.getPartitions(databaseName, tableName);
+        return PartitionListResponse.newBuilder()
+          .setState(OK)
+          .addAllPartition(partitions)
+          .build();
+      } catch (Throwable t) {
+        return PartitionListResponse.newBuilder()
+          .setState(returnError(t))
+          .build();
+      }
+    }
+
+    @Override
     public IndexResponse getIndexWithName(RpcController controller, SessionedStringProto
request)
         throws ServiceException {
       try {


Mime
View raw message