tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From blrun...@apache.org
Subject git commit: TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa)
Date Wed, 11 Dec 2013 06:16:48 GMT
Updated Branches:
  refs/heads/master 01c25d28c -> 93bae895f


TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/93bae895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/93bae895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/93bae895

Branch: refs/heads/master
Commit: 93bae895f539ca1c787150210ce78b2d3233d366
Parents: 01c25d2
Author: blrunner <jhjung@gruter.com>
Authored: Wed Dec 11 15:16:31 2013 +0900
Committer: blrunner <jhjung@gruter.com>
Committed: Wed Dec 11 15:16:31 2013 +0900

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../tajo-catalog-drivers/tajo-hcatalog/pom.xml  | 33 +++++++++++++++++---
 .../tajo/catalog/store/HCatalogStore.java       | 23 ++++++++++++--
 3 files changed, 50 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b32a5a5..1e662ef 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -86,6 +86,8 @@ Release 0.8.0 - unreleased
 
   IMPROVEMENTS
 
+    TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa)
+
     TAJO-336: Separate catalog stores into separate modules. (jaehwa)
 
     TAJO-329: Implement physical operator to store in column-partitioned table. (jaehwa)

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
index 738a9f7..aeb4a08 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
@@ -94,10 +94,7 @@
       <groupId>org.apache.tajo</groupId>
       <artifactId>tajo-rpc</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.tajo</groupId>
-      <artifactId>tajo-algebra</artifactId>
-    </dependency>
+
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
@@ -121,7 +118,6 @@
     </dependency>
   </dependencies>
 
-
   <profiles>
     <profile>
       <id>hcatalog-0.11.0</id>
@@ -368,6 +364,33 @@
       </dependencies>
     </profile>
     <profile>
+      <id>docs</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <executions>
+              <execution>
+                <!-- build javadoc jars per jar for publishing to maven -->
+                <id>module-javadocs</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>jar</goal>
+                </goals>
+                <configuration>
+                  <destDir>${project.build.directory}</destDir>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
       <id>src</id>
       <activation>
         <activeByDefault>false</activeByDefault>

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index d8a0e98..c41a690 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -30,7 +30,7 @@ import org.apache.hcatalog.data.Pair;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.partition.Partitions;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.common.TajoDataTypes;
@@ -103,9 +103,10 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
     HiveMetaStoreClient client = null;
     Path path = null;
     CatalogProtos.StoreType storeType = null;
-    Schema schema = null;
+    org.apache.tajo.catalog.Schema schema = null;
     Options options = null;
     TableStats stats = null;
+    Partitions partitions = null;
 
     // get db name and table name.
     try {
@@ -132,7 +133,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
       }
 
       // convert hcatalog field schema into tajo field schema.
-      schema = new Schema();
+      schema = new org.apache.tajo.catalog.Schema();
       HCatSchema tableSchema = HCatUtil.getTableSchemaWithPtnCols(table);
       List<HCatFieldSchema> fieldSchemaList = tableSchema.getFields();
       for (HCatFieldSchema eachField : fieldSchemaList) {
@@ -174,6 +175,18 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
         }
       }
 
+      if (table.getPartitionKeys() != null) {
+        partitions = new Partitions();
+        List<FieldSchema> partitionKeys = table.getPartitionKeys();
+        for(int i = 0; i < partitionKeys.size(); i++) {
+          FieldSchema fieldSchema = partitionKeys.get(i);
+          TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(fieldSchema.getType().toString());
+          partitions.addColumn(new Column(fieldSchema.getName(), dataType));
+        }
+        partitions.setPartitionsType(CatalogProtos.PartitionsType.COLUMN);
+      }
+
+
     } finally {
       HCatUtil.closeHiveClientQuietly(client);
     }
@@ -183,10 +196,14 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
     if (stats != null) {
       tableDesc.setStats(stats);
     }
+    if (partitions != null) {
+      tableDesc.setPartitions(partitions);
+    }
 
     return tableDesc;
   }
 
+
   private TajoDataTypes.Type getDataType(final String typeStr) {
     try {
       return Enum.valueOf(TajoDataTypes.Type.class, typeStr);


Mime
View raw message