Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D52E610EE7 for ; Sat, 28 Dec 2013 06:36:42 +0000 (UTC) Received: (qmail 36674 invoked by uid 500); 28 Dec 2013 06:36:34 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 36578 invoked by uid 500); 28 Dec 2013 06:36:30 -0000 Mailing-List: contact commits-help@tajo.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.incubator.apache.org Delivered-To: mailing list commits@tajo.incubator.apache.org Received: (qmail 36496 invoked by uid 99); 28 Dec 2013 06:36:21 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 28 Dec 2013 06:36:21 +0000 X-ASF-Spam-Status: No, hits=-2000.5 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Sat, 28 Dec 2013 06:36:20 +0000 Received: (qmail 36348 invoked by uid 99); 28 Dec 2013 06:35:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 28 Dec 2013 06:35:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id F154D8806FE; Sat, 28 Dec 2013 06:35:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jihoonson@apache.org To: commits@tajo.incubator.apache.org Date: Sat, 28 Dec 2013 06:35:59 -0000 Message-Id: In-Reply-To: <6671755524c04aa186032343e3ec77e5@git.apache.org> References: <6671755524c04aa186032343e3ec77e5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/50] [abbrv] git commit: TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa) X-Virus-Checked: Checked by ClamAV on apache.org TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa) Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/93bae895 Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/93bae895 Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/93bae895 Branch: refs/heads/DAG-execplan Commit: 93bae895f539ca1c787150210ce78b2d3233d366 Parents: 01c25d2 Author: blrunner Authored: Wed Dec 11 15:16:31 2013 +0900 Committer: blrunner Committed: Wed Dec 11 15:16:31 2013 +0900 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 33 +++++++++++++++++--- .../tajo/catalog/store/HCatalogStore.java | 23 ++++++++++++-- 3 files changed, 50 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b32a5a5..1e662ef 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -86,6 +86,8 @@ Release 0.8.0 - unreleased IMPROVEMENTS + TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa) + TAJO-336: Separate catalog stores into separate modules. (jaehwa) TAJO-329: Implement physical operator to store in column-partitioned table. (jaehwa) http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml index 738a9f7..aeb4a08 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml @@ -94,10 +94,7 @@ org.apache.tajo tajo-rpc - - org.apache.tajo - tajo-algebra - + com.google.protobuf protobuf-java @@ -121,7 +118,6 @@ - hcatalog-0.11.0 @@ -368,6 +364,33 @@ + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + src false http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index d8a0e98..c41a690 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -30,7 +30,7 @@ import org.apache.hcatalog.data.Pair; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.partition.Partitions; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; @@ -103,9 +103,10 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { HiveMetaStoreClient client = null; Path path = null; CatalogProtos.StoreType storeType = null; - Schema schema = null; + org.apache.tajo.catalog.Schema schema = null; Options options = null; TableStats stats = null; + Partitions partitions = null; // get db name and table name. try { @@ -132,7 +133,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } // convert hcatalog field schema into tajo field schema. - schema = new Schema(); + schema = new org.apache.tajo.catalog.Schema(); HCatSchema tableSchema = HCatUtil.getTableSchemaWithPtnCols(table); List fieldSchemaList = tableSchema.getFields(); for (HCatFieldSchema eachField : fieldSchemaList) { @@ -174,6 +175,18 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } } + if (table.getPartitionKeys() != null) { + partitions = new Partitions(); + List partitionKeys = table.getPartitionKeys(); + for(int i = 0; i < partitionKeys.size(); i++) { + FieldSchema fieldSchema = partitionKeys.get(i); + TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(fieldSchema.getType().toString()); + partitions.addColumn(new Column(fieldSchema.getName(), dataType)); + } + partitions.setPartitionsType(CatalogProtos.PartitionsType.COLUMN); + } + + } finally { HCatUtil.closeHiveClientQuietly(client); } @@ -183,10 +196,14 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { if (stats != null) { tableDesc.setStats(stats); } + if (partitions != null) { + tableDesc.setPartitions(partitions); + } return tableDesc; } + private TajoDataTypes.Type getDataType(final String typeStr) { try { return Enum.valueOf(TajoDataTypes.Type.class, typeStr);