drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [1/3] drill git commit: DRILL-4161: Make Hive Metastore client caching policy user configurable.
Date Fri, 11 Dec 2015 18:25:13 GMT
Repository: drill
Updated Branches:
  refs/heads/master 539cbba56 -> bb3fc1521


DRILL-4161: Make Hive Metastore client caching policy user configurable.

Remove common code.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bb3fc152
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bb3fc152
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bb3fc152

Branch: refs/heads/master
Commit: bb3fc15216d9cab804fc9a6f0e5bd34597dd4394
Parents: 02ce44b
Author: Jinfeng Ni <jni@apache.org>
Authored: Mon Dec 7 14:17:04 2015 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Fri Dec 11 09:49:55 2015 -0800

----------------------------------------------------------------------
 .../store/hive/DrillHiveMetaStoreClient.java    | 45 ++++++++++++++------
 1 file changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/bb3fc152/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java
index 2cce699..8b46a93 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java
@@ -53,6 +53,11 @@ import java.util.concurrent.TimeUnit;
 public abstract class DrillHiveMetaStoreClient extends HiveMetaStoreClient {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveMetaStoreClient.class);
 
+  public final String HIVE_METASTORE_CACHE_TTL = "hive.metastore.cache-ttl-seconds";
+  public final String HIVE_METASTORE_CACHE_EXPIRE = "hive.metastore.cache-expire-after";
+  public final String HIVE_METASTORE_CACHE_EXPIRE_AFTER_WRITE = "write";
+  public final String HIVE_METASTORE_CACHE_EXPIRE_AFTER_ACCESS = "access";
+
   protected final Map<String, String> hiveConfigOverride;
 
   protected final LoadingCache<String, List<String>> databases;
@@ -137,22 +142,38 @@ public abstract class DrillHiveMetaStoreClient extends HiveMetaStoreClient
{
   private DrillHiveMetaStoreClient(final HiveConf hiveConf, final Map<String, String>
hiveConfigOverride)
       throws MetaException {
     super(hiveConf);
+
+    int hmsCacheTTL = 60; // default is 60 seconds
+    boolean expireAfterWrite = true; // default is expire after write.
+
+    if (hiveConfigOverride.containsKey(HIVE_METASTORE_CACHE_TTL)) {
+      hmsCacheTTL = Integer.valueOf(hiveConfigOverride.get(HIVE_METASTORE_CACHE_TTL));
+      logger.warn("Hive metastore cache ttl is set to {} seconds.", hmsCacheTTL);
+    }
+
+    if (hiveConfigOverride.containsKey(HIVE_METASTORE_CACHE_EXPIRE)) {
+      if (hiveConfigOverride.get(HIVE_METASTORE_CACHE_EXPIRE).equalsIgnoreCase(HIVE_METASTORE_CACHE_EXPIRE_AFTER_WRITE))
{
+        expireAfterWrite = true;
+      } else if (hiveConfigOverride.get(HIVE_METASTORE_CACHE_EXPIRE).equalsIgnoreCase(HIVE_METASTORE_CACHE_EXPIRE_AFTER_ACCESS))
{
+        expireAfterWrite = false;
+      }
+      logger.warn("Hive metastore cache expire policy is set to {}", expireAfterWrite? "expireAfterWrite"
: "expireAfterAccess");
+    }
+
     this.hiveConfigOverride = hiveConfigOverride;
 
-    databases = CacheBuilder //
-        .newBuilder() //
-        .expireAfterWrite(1, TimeUnit.MINUTES) //
-        .build(new DatabaseLoader());
+    final CacheBuilder cacheBuilder = CacheBuilder
+        .newBuilder();
 
-    tableNameLoader = CacheBuilder //
-        .newBuilder() //
-        .expireAfterWrite(1, TimeUnit.MINUTES) //
-        .build(new TableNameLoader());
+    if (expireAfterWrite) {
+      cacheBuilder.expireAfterWrite(hmsCacheTTL, TimeUnit.SECONDS);
+    } else {
+      cacheBuilder.expireAfterAccess(hmsCacheTTL, TimeUnit.SECONDS);
+    }
 
-    tableLoaders = CacheBuilder //
-        .newBuilder() //
-        .expireAfterWrite(1, TimeUnit.MINUTES) //
-        .build(new TableLoader());
+    databases = cacheBuilder.build(new DatabaseLoader());
+    tableNameLoader = cacheBuilder.build(new TableNameLoader());
+    tableLoaders = cacheBuilder.build(new TableLoader());
 
   }
 


Mime
View raw message