hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [11/23] hive git commit: HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan reviewed by Prasanth Jayachandran)
Date Fri, 13 Oct 2017 00:07:02 GMT
HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan reviewed
by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f1050a6e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f1050a6e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f1050a6e

Branch: refs/heads/hive-14535
Commit: f1050a6e8462864a3d8c9696943cf5b4d12d9515
Parents: 63bda8c
Author: Prasanth Jayachandran <prasanthj@apache.org>
Authored: Wed Oct 11 14:29:57 2017 -0700
Committer: Prasanth Jayachandran <prasanthj@apache.org>
Committed: Wed Oct 11 14:29:57 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  3 +
 .../hive/ql/io/sarg/ConvertAstToSearchArg.java  | 59 ++++++++++++++++++--
 2 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f1050a6e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 942629b..9084785 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1393,6 +1393,9 @@ public class HiveConf extends Configuration {
         "references for the cached object. Setting this to true can help avoid out of memory\n"
+
         "issues under memory pressure (in some cases) at the cost of slight unpredictability
in\n" +
         "overall query performance."),
+    HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB("hive.io.sarg.cache.max.weight.mb", 10,
+        "The max weight allowed for the SearchArgument Cache. By default, the cache allows
a max-weight of 10MB, " +
+        "after which entries will be evicted."),
 
     HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL("hive.lazysimple.extended_boolean_literal",
false,
         "LazySimpleSerde uses this property to determine if it treats 'T', 't', 'F', 'f',\n"
+

http://git-wip-us.apache.org/repos/asf/hive/blob/f1050a6e/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
index 997334b..b9bf801 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
@@ -21,13 +21,16 @@ package org.apache.hadoop.hive.ql.io.sarg;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hadoop.hive.ql.io.sarg.LiteralDelegate;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -436,14 +439,60 @@ public class ConvertAstToSearchArg {
     }
   }
 
-
   public static final String SARG_PUSHDOWN = "sarg.pushdown";
 
+  private static volatile Cache<String, SearchArgument> sargsCache = null;
+
+  private static synchronized Cache<String, SearchArgument> initializeAndGetSargsCache(Configuration
conf) {
+    if (sargsCache == null) {
+      sargsCache = CacheBuilder.newBuilder()
+            .weigher((String key, SearchArgument value) -> key.length())
+            .maximumWeight(
+                HiveConf.getIntVar(conf,
+                                   HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) *
1024 *1024
+            )
+            .build(); // Can't use CacheLoader because SearchArguments may be built either
from Kryo strings,
+                      // or from expressions.
+    }
+    return sargsCache;
+  }
+
+  private static Cache<String, SearchArgument> getSargsCache(Configuration conf) {
+    return sargsCache == null? initializeAndGetSargsCache(conf) : sargsCache;
+  }
+
+  private static boolean isSargsCacheEnabled(Configuration conf) {
+    return HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) >
0;
+  }
+
+  private static SearchArgument getSearchArgumentFromString(Configuration conf, String sargString)
{
+
+    try {
+      return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, () -> create(sargString))
+                                      : create(sargString);
+    }
+    catch (ExecutionException exception) {
+      throw new RuntimeException(exception);
+    }
+  }
+
+  private static SearchArgument getSearchArgumentFromExpression(Configuration conf, String
sargString) {
+
+    try {
+      return isSargsCacheEnabled(conf)?
+              getSargsCache(conf).get(sargString,
+                                      () -> create(conf, SerializationUtilities.deserializeExpression(sargString)))
+              : create(conf, SerializationUtilities.deserializeExpression(sargString));
+    }
+    catch (ExecutionException exception) {
+      throw new RuntimeException(exception);
+    }
+  }
+
   public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression)
{
     return new ConvertAstToSearchArg(conf, expression).buildSearchArgument();
   }
 
-
   private final static ThreadLocal<Kryo> kryo = new ThreadLocal<Kryo>() {
     protected Kryo initialValue() { return new Kryo(); }
   };
@@ -459,9 +508,9 @@ public class ConvertAstToSearchArg {
   public static SearchArgument createFromConf(Configuration conf) {
     String sargString;
     if ((sargString = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR)) != null) {
-      return create(conf, SerializationUtilities.deserializeExpression(sargString));
+      return getSearchArgumentFromExpression(conf, sargString);
     } else if ((sargString = conf.get(SARG_PUSHDOWN)) != null) {
-      return create(sargString);
+      return getSearchArgumentFromString(conf, sargString);
     }
     return null;
   }


Mime
View raw message