hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mit...@apache.org
Subject [2/2] hive git commit: HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan reviewed by Prasanth Jayachandran)
Date Thu, 12 Oct 2017 17:45:50 GMT
HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan reviewed
by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c332103e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c332103e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c332103e

Branch: refs/heads/branch-2
Commit: c332103ef3254f15dca1077335191ecd96e65c38
Parents: 58fa65a
Author: Mithun Radhakrishnan <mithun@apache.org>
Authored: Thu Oct 12 10:41:38 2017 -0700
Committer: Mithun Radhakrishnan <mithun@apache.org>
Committed: Thu Oct 12 10:41:38 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  3 +
 .../hive/ql/io/sarg/ConvertAstToSearchArg.java  | 76 ++++++++++++++++++--
 2 files changed, 74 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c332103e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 62eabbc..3d3f3b2 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1382,6 +1382,9 @@ public class HiveConf extends Configuration {
         "references for the cached object. Setting this to true can help avoid out of memory\n"
+
         "issues under memory pressure (in some cases) at the cost of slight unpredictability
in\n" +
         "overall query performance."),
+    HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB("hive.io.sarg.cache.max.weight.mb", 10,
+        "The max weight allowed for the SearchArgument Cache. By default, the cache allows
a max-weight of 10MB, " +
+        "after which entries will be evicted."),
 
     HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL("hive.lazysimple.extended_boolean_literal",
false,
         "LazySimpleSerde uses this property to determine if it treats 'T', 't', 'F', 'f',\n"
+

http://git-wip-us.apache.org/repos/asf/hive/blob/c332103e/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
index 997334b..2de8319 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
@@ -21,13 +21,18 @@ package org.apache.hadoop.hive.ql.io.sarg;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hadoop.hive.ql.io.sarg.LiteralDelegate;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -436,14 +441,75 @@ public class ConvertAstToSearchArg {
     }
   }
 
-
   public static final String SARG_PUSHDOWN = "sarg.pushdown";
 
+  private static volatile Cache<String, SearchArgument> sargsCache = null;
+
+  private static synchronized Cache<String, SearchArgument> initializeAndGetSargsCache(Configuration
conf) {
+    if (sargsCache == null) {
+      sargsCache = CacheBuilder.newBuilder()
+            .weigher(new Weigher<String, SearchArgument>() {
+              @Override
+              public int weigh(String key, SearchArgument value) {
+                return key.length();
+              }
+            })
+            .maximumWeight(
+                HiveConf.getIntVar(conf,
+                                   HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) *
1024 *1024
+            )
+            .build(); // Can't use CacheLoader because SearchArguments may be built either
from Kryo strings,
+                      // or from expressions.
+    }
+    return sargsCache;
+  }
+
+  private static Cache<String, SearchArgument> getSargsCache(Configuration conf) {
+    return sargsCache == null? initializeAndGetSargsCache(conf) : sargsCache;
+  }
+
+  private static boolean isSargsCacheEnabled(Configuration conf) {
+    return HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) >
0;
+  }
+
+  private static SearchArgument getSearchArgumentFromString(Configuration conf, final String
sargString) {
+
+    try {
+      return isSargsCacheEnabled(conf)?
+          getSargsCache(conf).get(sargString, new Callable<SearchArgument>() {
+            @Override
+            public SearchArgument call() {
+              return create(sargString);
+            }
+          })
+          : create(sargString);
+    }
+    catch (ExecutionException exception) {
+      throw new RuntimeException(exception);
+    }
+  }
+
+  private static SearchArgument getSearchArgumentFromExpression(final Configuration conf,
final String sargString) {
+
+    try {
+      return isSargsCacheEnabled(conf)?
+              getSargsCache(conf).get(sargString, new Callable<SearchArgument>() {
+                @Override
+                public SearchArgument call() {
+                  return create(conf, SerializationUtilities.deserializeExpression(sargString));
+                }
+              })
+              : create(conf, SerializationUtilities.deserializeExpression(sargString));
+    }
+    catch (ExecutionException exception) {
+      throw new RuntimeException(exception);
+    }
+  }
+
   public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression)
{
     return new ConvertAstToSearchArg(conf, expression).buildSearchArgument();
   }
 
-
   private final static ThreadLocal<Kryo> kryo = new ThreadLocal<Kryo>() {
     protected Kryo initialValue() { return new Kryo(); }
   };
@@ -459,9 +525,9 @@ public class ConvertAstToSearchArg {
   public static SearchArgument createFromConf(Configuration conf) {
     String sargString;
     if ((sargString = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR)) != null) {
-      return create(conf, SerializationUtilities.deserializeExpression(sargString));
+      return getSearchArgumentFromExpression(conf, sargString);
     } else if ((sargString = conf.get(SARG_PUSHDOWN)) != null) {
-      return create(sargString);
+      return getSearchArgumentFromString(conf, sargString);
     }
     return null;
   }


Mime
View raw message