Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DCF6A18C07 for ; Mon, 27 Jul 2015 21:14:20 +0000 (UTC) Received: (qmail 50679 invoked by uid 500); 27 Jul 2015 21:07:39 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 50635 invoked by uid 500); 27 Jul 2015 21:07:39 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 50624 invoked by uid 99); 27 Jul 2015 21:07:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Jul 2015 21:07:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 92A31E0AFA; Mon, 27 Jul 2015 21:07:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Message-Id: <1d0f653858014605a70a3ce2287de2f3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-11305 : LLAP: Hybrid Map-join cache returns invalid data (Sergey Shelukhin, reviewed by Gopal V) Date: Mon, 27 Jul 2015 21:07:38 +0000 (UTC) Repository: hive Updated Branches: refs/heads/llap 1e3b59d37 -> 6bdb903e4 HIVE-11305 : LLAP: Hybrid Map-join cache returns invalid data (Sergey Shelukhin, reviewed by Gopal V) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bdb903e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bdb903e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bdb903e Branch: refs/heads/llap Commit: 6bdb903e49bc8271ff52097dcf87ec8e12b8304d Parents: 1e3b59d Author: Sergey Shelukhin Authored: Mon Jul 27 14:07:27 2015 -0700 Committer: Sergey Shelukhin Committed: Mon Jul 27 14:07:27 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/MapJoinOperator.java | 8 ++++++-- .../apache/hadoop/hive/ql/exec/ObjectCacheFactory.java | 10 ++++++---- .../hadoop/hive/ql/exec/tez/LlapObjectCache.java | 13 ++++++------- .../vector/mapjoin/VectorMapJoinCommonOperator.java | 2 -- 4 files changed, 18 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6bdb903e/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 31c5723..ed75639 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -134,7 +134,6 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem // the task in. On MR: The cache is a no-op. String queryId = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVEQUERYID); cacheKey = "HASH_MAP_" + this.getOperatorId() + "_container"; - cache = ObjectCacheFactory.getCache(hconf, queryId); loader = getHashTableLoader(hconf); @@ -196,10 +195,15 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem if (!loadCalled && spilled) { // we can't use the cached table because it has spilled. + loadHashTable(getExecContext(), MapredContext.get()); } else { if (LOG.isInfoEnabled()) { - LOG.info("Using tables from cache: " + pair.getLeft()); + String s = "Using tables from cache: ["; + for (MapJoinTableContainer c : pair.getLeft()) { + s += ((c == null) ? "null" : c.getClass().getSimpleName()) + ", "; + } + LOG.info(s + "]"); } // let's use the table from the cache. mapJoinTables = pair.getLeft(); http://git-wip-us.apache.org/repos/asf/hive/blob/6bdb903e/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java index dcf16f9..22853bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java @@ -48,6 +48,7 @@ public class ObjectCacheFactory { if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { if (LlapIoProxy.isDaemon()) { // daemon if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_OBJECT_CACHE_ENABLED)) { + // LLAP object cache, unlike others, does not use globals. Thus, get the existing one. return getLlapObjectCache(queryId); } else { // no cache return new ObjectCacheWrapper( @@ -66,19 +67,20 @@ public class ObjectCacheFactory { private static ObjectCache getLlapObjectCache(String queryId) { // If order of events (i.e. dagstart and fragmentstart) was guaranteed, we could just // create the cache when dag starts, and blindly return it to execution here. + if (queryId == null) throw new RuntimeException("Query ID cannot be null"); ObjectCache result = llapQueryCaches.get(queryId); if (result != null) return result; result = new LlapObjectCache(); ObjectCache old = llapQueryCaches.putIfAbsent(queryId, result); - if (old == null && LOG.isDebugEnabled()) { - LOG.debug("Created object cache for " + queryId); + if (old == null && LOG.isInfoEnabled()) { + LOG.info("Created object cache for " + queryId); } return (old != null) ? old : result; } public static void removeLlapQueryCache(String queryId) { - if (LOG.isDebugEnabled()) { - LOG.debug("Removing object cache for " + queryId); + if (LOG.isInfoEnabled()) { + LOG.info("Removing object cache for " + queryId); } llapQueryCaches.remove(queryId); } http://git-wip-us.apache.org/repos/asf/hive/blob/6bdb903e/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectCache.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectCache.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectCache.java index 00f3c54..a7936a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectCache.java @@ -42,17 +42,16 @@ public class LlapObjectCache implements org.apache.hadoop.hive.ql.exec.ObjectCac private static final Log LOG = LogFactory.getLog(LlapObjectCache.class.getName()); - private static final Cache registry - = CacheBuilder.newBuilder().softValues().build(); + private static ExecutorService staticPool = Executors.newCachedThreadPool(); - private static final Map locks - = new HashMap(); + private static final boolean isLogInfoEnabled = LOG.isInfoEnabled(); - private static final ReentrantLock lock = new ReentrantLock(); + private final Cache registry = CacheBuilder.newBuilder().softValues().build(); - private static ExecutorService staticPool = Executors.newCachedThreadPool(); + private final Map locks = new HashMap(); + + private final ReentrantLock lock = new ReentrantLock(); - private static final boolean isLogInfoEnabled = LOG.isInfoEnabled(); @Override public void release(String key) { http://git-wip-us.apache.org/repos/asf/hive/blob/6bdb903e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java index 63f664d..81ee41b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java @@ -530,7 +530,6 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { - VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; @@ -621,7 +620,6 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); - HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: {