drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [3/3] drill git commit: DRILL-4127: Reduce Hive metastore client API call in HiveSchema.
Date Fri, 11 Dec 2015 18:25:15 GMT
DRILL-4127: Reduce Hive metastore client API call in HiveSchema.

1) Use lazy loading of tableNames in HiveSchema, in stead of pre-loading all table names under
each HiveSchema.
2) Do not call get_all_databases for subSchema to check existence if the name comes from getSubSchemaNames()
directly.

review comments.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b8b18ebf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b8b18ebf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b8b18ebf

Branch: refs/heads/master
Commit: b8b18ebf4b9dbf2027d1f5e9666504dbe11901cb
Parents: 539cbba
Author: Jinfeng Ni <jni@apache.org>
Authored: Wed Nov 18 20:18:51 2015 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Fri Dec 11 09:49:55 2015 -0800

----------------------------------------------------------------------
 .../store/hive/schema/HiveDatabaseSchema.java   | 24 ++++++++++++++++----
 .../store/hive/schema/HiveSchemaFactory.java    | 22 ++++++++----------
 2 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/b8b18ebf/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
index 48c034e..6f43639 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
@@ -17,30 +17,36 @@
  */
 package org.apache.drill.exec.store.hive.schema;
 
-import java.util.List;
 import java.util.Set;
 
 import org.apache.calcite.schema.Table;
 
 import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaConfig;
+import org.apache.drill.exec.store.hive.DrillHiveMetaStoreClient;
 import org.apache.drill.exec.store.hive.HiveStoragePluginConfig;
 import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory.HiveSchema;
 
 import com.google.common.collect.Sets;
+import org.apache.thrift.TException;
 
 public class HiveDatabaseSchema extends AbstractSchema{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class);
 
   private final HiveSchema hiveSchema;
-  private final Set<String> tables;
+  private Set<String> tables;
+  private final DrillHiveMetaStoreClient mClient;
+  private final SchemaConfig schemaConfig;
 
   public HiveDatabaseSchema( //
-      List<String> tableList, //
       HiveSchema hiveSchema, //
-      String name) {
+      String name,
+      DrillHiveMetaStoreClient mClient,
+      SchemaConfig schemaConfig) {
     super(hiveSchema.getSchemaPath(), name);
     this.hiveSchema = hiveSchema;
-    this.tables = Sets.newHashSet(tableList);
+    this.mClient = mClient;
+    this.schemaConfig = schemaConfig;
   }
 
   @Override
@@ -50,6 +56,14 @@ public class HiveDatabaseSchema extends AbstractSchema{
 
   @Override
   public Set<String> getTableNames() {
+    if (tables == null) {
+      try {
+        tables = Sets.newHashSet(mClient.getTableNames(this.name, schemaConfig.getIgnoreAuthErrors()));
+      } catch (final TException e) {
+        logger.warn("Failure while attempting to access HiveDatabase '{}'.", this.name, e.getCause());
+        tables = Sets.newHashSet(); // empty set.
+      }
+    }
     return tables;
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/b8b18ebf/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index 73e7bf7..16aca54 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -120,15 +120,13 @@ public class HiveSchemaFactory implements SchemaFactory {
 
     @Override
     public AbstractSchema getSubSchema(String name) {
-      List<String> tables;
       try {
-        List<String> dbs = mClient.getDatabases();
+        List<String> dbs = mClient.getDatabases(schemaConfig.getIgnoreAuthErrors());
         if (!dbs.contains(name)) {
           logger.debug("Database '{}' doesn't exists in Hive storage '{}'", name, schemaName);
           return null;
         }
-        tables = mClient.getTableNames(name);
-        HiveDatabaseSchema schema = new HiveDatabaseSchema(tables, this, name);
+        HiveDatabaseSchema schema = getSubSchemaKnownExists(name);
         if (name.equals("default")) {
           this.defaultSchema = schema;
         }
@@ -137,12 +135,17 @@ public class HiveSchemaFactory implements SchemaFactory {
         logger.warn("Failure while attempting to access HiveDatabase '{}'.", name, e.getCause());
         return null;
       }
+    }
 
+    /** Help method to get subschema when we know it exists (already checks the existence)
*/
+    private HiveDatabaseSchema getSubSchemaKnownExists(String name) {
+      HiveDatabaseSchema schema = new HiveDatabaseSchema(this, name, mClient, schemaConfig);
+      return schema;
     }
 
     void setHolder(SchemaPlus plusOfThis) {
       for (String s : getSubSchemaNames()) {
-        plusOfThis.add(s, getSubSchema(s));
+        plusOfThis.add(s, getSubSchemaKnownExists(s));
       }
     }
 
@@ -154,7 +157,7 @@ public class HiveSchemaFactory implements SchemaFactory {
     @Override
     public Set<String> getSubSchemaNames() {
       try {
-        List<String> dbs = mClient.getDatabases();
+        List<String> dbs = mClient.getDatabases(schemaConfig.getIgnoreAuthErrors());
         return Sets.newHashSet(dbs);
       } catch (final TException e) {
         logger.warn("Failure while getting Hive database list.", e);
@@ -215,13 +218,6 @@ public class HiveSchemaFactory implements SchemaFactory {
     public String getTypeName() {
       return HiveStoragePluginConfig.NAME;
     }
-
-    @Override
-    public void close() throws Exception {
-      if (mClient != null) {
-        mClient.close();
-      }
-    }
   }
 
 }


Mime
View raw message