tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [2/3] git commit: TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
Date Wed, 13 Aug 2014 00:47:04 GMT
TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)

Closes #108


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fbf71668
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fbf71668
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fbf71668

Branch: refs/heads/index_support
Commit: fbf7166897b3b25d8319198216dc4ca529e677ce
Parents: 87e7ba2
Author: jinossy <jinossy@gmail.com>
Authored: Mon Aug 11 15:18:29 2014 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Mon Aug 11 15:18:29 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../tajo/catalog/store/HCatalogStore.java       | 11 ++++++-----
 .../catalog/store/HCatalogStoreClientPool.java  | 20 ++++++++++++++++----
 .../apache/tajo/catalog/store/HCatalogUtil.java |  6 ++++++
 4 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8788b17..839125e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -109,6 +109,8 @@ Release 0.9.0 - unreleased
 
   BUG FIXES
 
+    TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
+
     TAJO-947: ColPartitionStoreExec can cause URISyntaxException due 
     to special characters. (Mai Hai Thanh via hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 61bdccd..fb8f4c4 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -48,6 +48,7 @@ import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.exception.InternalException;
 import org.apache.tajo.storage.StorageConstants;
 import org.apache.tajo.util.KeyValueSet;
+import org.apache.thrift.TException;
 
 import java.io.IOException;
 import java.util.*;
@@ -77,13 +78,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
   @Override
   public boolean existTable(final String databaseName, final String tableName) throws CatalogException
{
     boolean exist = false;
-    org.apache.hadoop.hive.ql.metadata.Table table = null;
+    org.apache.hadoop.hive.ql.metadata.Table table;
     HCatalogStoreClientPool.HCatalogStoreClient client = null;
 
     // get table
     try {
       client = clientPool.getClient();
-      table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+      table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
       if (table != null) {
         exist = true;
       }
@@ -118,7 +119,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
       // get hive table schema
       try {
         client = clientPool.getClient();
-        table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+        table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
         path = table.getPath();
       } catch (NoSuchObjectException nsoe) {
         throw new CatalogException("Table not found. - tableName:" + tableName, nsoe);
@@ -291,7 +292,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
     try {
       client = clientPool.getClient();
       return client.getHiveClient().getAllTables(databaseName);
-    } catch (MetaException e) {
+    } catch (TException e) {
       throw new CatalogException(e);
     } finally {
       if(client != null) client.release();
@@ -401,7 +402,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
     try {
       client = clientPool.getClient();
       return client.getHiveClient().getAllDatabases();
-    } catch (MetaException e) {
+    } catch (TException e) {
       throw new CatalogException(e);
     } finally {
       if (client != null) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
index 2fff67c..8ccb100 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
@@ -16,7 +16,9 @@ package org.apache.tajo.catalog.store;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.*;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.log4j.Logger;
 
 import java.util.Iterator;
@@ -41,12 +43,22 @@ public class HCatalogStoreClientPool {
    * connection pool.
    */
   public class HCatalogStoreClient {
-    private final HiveMetaStoreClient hiveClient;
+    private final IMetaStoreClient hiveClient;
     public AtomicBoolean isInUse = new AtomicBoolean(false);
 
     private HCatalogStoreClient(HiveConf hiveConf) {
       try {
-        this.hiveClient = new HiveMetaStoreClient(hiveConf);
+        HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+          @Override
+          public HiveMetaHook getHook(Table table) throws MetaException {
+            /* metadata hook implementation, or null if this
+             * storage handler does not need any metadata notifications
+             */
+            return null;
+          }
+        };
+
+        this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
         clientPool.add(this);
         LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")");
       } catch (Exception e) {
@@ -58,7 +70,7 @@ public class HCatalogStoreClientPool {
     /**
      * Returns the internal HiveMetaStoreClient object.
      */
-    public HiveMetaStoreClient getHiveClient() {
+    public IMetaStoreClient getHiveClient() {
       return hiveClient;
     }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index 9e60768..54fdb9d 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -20,9 +20,11 @@ package org.apache.tajo.catalog.store;
 import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
 import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hcatalog.common.HCatException;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -30,6 +32,7 @@ import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.tajo.catalog.exception.CatalogException;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.common.TajoDataTypes;
+import org.apache.thrift.TException;
 import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
 
 public class HCatalogUtil {
@@ -137,4 +140,7 @@ public class HCatalogUtil {
     }
   }
 
+  public static Table getTable(IMetaStoreClient client, String dbName, String tableName)
throws TException {
+    return new Table(client.getTable(dbName, tableName));
+  }
 }


Mime
View raw message