hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [3/3] hive git commit: HIVE-16318 : LLAP cache: address some issues in 2.2/2.3 (Sergey Shelukhin, reviewed by Siddharth Seth and Prasanth Jayachandran)
Date Wed, 29 Mar 2017 19:44:10 GMT
HIVE-16318 : LLAP cache: address some issues in 2.2/2.3 (Sergey Shelukhin, reviewed by Siddharth
Seth and Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae3fef4d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae3fef4d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae3fef4d

Branch: refs/heads/branch-2.2
Commit: ae3fef4d0e1237eecd5aa71d771e290ba30811c1
Parents: f31f749
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Mar 29 12:09:48 2017 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Mar 29 12:44:00 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  8 ++++-
 .../llap/cache/LowLevelCacheMemoryManager.java  |  7 ----
 .../llap/cache/LowLevelFifoCachePolicy.java     |  2 +-
 .../llap/cache/LowLevelLrfuCachePolicy.java     |  6 ----
 .../hive/llap/io/api/impl/LlapIoImpl.java       | 37 ++++++++++++++++----
 5 files changed, 39 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index de2e9d6..ce5ec43 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2868,13 +2868,19 @@ public class HiveConf extends Configuration {
         "LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n"
+
         "custom off-heap allocator, 'none' doesn't use either (this mode may result in\n"
+
         "significant performance degradation)"),
-    LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "16Kb", new SizeValidator(),
+    LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "256Kb", new SizeValidator(),
         "Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n"
+
         "padded to minimum allocation. For ORC, should generally be the same as the expected\n"
+
         "compression buffer size, or next lowest power of 2. Must be a power of 2."),
     LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", "16Mb", new SizeValidator(),
         "Maximum allocation possible from LLAP buddy allocator. For ORC, should be as large
as\n" +
         "the largest expected ORC compression buffer size. Must be a power of 2."),
+    @Deprecated
+    LLAP_IO_METADATA_FRACTION("hive.llap.io.metadata.fraction", 0.1f,
+        "Temporary setting for on-heap metadata cache fraction of xmx, set to avoid potential\n"
+
+        "heap problems on very large datasets when on-heap metadata cache takes over\n" +
+        "everything. -1 managed metadata and data together (which is more flexible). This\n"
+
+        "setting will be removed (in effect become -1) once ORC metadata cache is moved off-heap."),
     LLAP_ALLOCATOR_ARENA_COUNT("hive.llap.io.allocator.arena.count", 8,
         "Arena count for LLAP low-level cache; cache will be allocated in the steps of\n"
+
         "(size/arena_count) bytes. This size must be <= 1Gb and >= max allocation;
if it is\n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 88bfa8b..91545ed 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -38,18 +38,11 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
   private long maxSize;
 
   public LowLevelCacheMemoryManager(
-      Configuration conf, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
-    this(HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), evictor, metrics);
-  }
-
-  @VisibleForTesting
-  public LowLevelCacheMemoryManager(
       long maxSize, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
     this.maxSize = maxSize;
     this.evictor = evictor;
     this.usedMemory = new AtomicLong(0);
     this.metrics = metrics;
-    metrics.setCacheCapacityTotal(maxSize);
     if (LlapIoImpl.LOG.isInfoEnabled()) {
       LlapIoImpl.LOG.info("Memory manager initialized with max size {} and" +
           " {} ability to evict blocks", maxSize, ((evictor == null) ? "no " : ""));

http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
index 430a5f8..8006502 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
@@ -34,7 +34,7 @@ public class LowLevelFifoCachePolicy implements LowLevelCachePolicy {
   private EvictionListener evictionListener;
   private LlapOomDebugDump parentDebugDump;
 
-  public LowLevelFifoCachePolicy(Configuration conf) {
+  public LowLevelFifoCachePolicy() {
     LlapIoImpl.LOG.info("FIFO cache policy");
     buffers = new LinkedList<LlapCacheableBuffer>();
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 4cd2c18..1bf5593 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -69,12 +69,6 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
   private EvictionListener evictionListener;
   private LlapOomDebugDump parentDebugDump;
 
-  public LowLevelLrfuCachePolicy(Configuration conf) {
-    this((int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
-        HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), conf);
-  }
-
-  @VisibleForTesting
   public LowLevelLrfuCachePolicy(int minBufferSize, long maxSize, Configuration conf) {
     lambda = HiveConf.getFloatVar(conf, HiveConf.ConfVars.LLAP_LRFU_LAMBDA);
     int maxBuffers = (int)Math.ceil((maxSize * 1.0) / minBufferSize);

http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index 7c309a4..5a86114 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
 
 import javax.management.ObjectName;
 
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
 import org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -121,11 +122,32 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch>
{
     if (useLowLevelCache) {
       // Memory manager uses cache policy to trigger evictions, so create the policy first.
       boolean useLrfu = HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_USE_LRFU);
-      LowLevelCachePolicy cachePolicy =
-          useLrfu ? new LowLevelLrfuCachePolicy(conf) : new LowLevelFifoCachePolicy(conf);
+      long totalMemorySize = HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+      int minAllocSize = (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+      float metadataFraction = HiveConf.getFloatVar(conf, ConfVars.LLAP_IO_METADATA_FRACTION);
+      long metaMem = 0;
+      // TODO: this split a workaround until HIVE-15665.
+      //       Technically we don't have to do it for on-heap data cache but we'd do for
testing.
+      boolean isSplitCache = metadataFraction > 0f;
+      if (isSplitCache) {
+        metaMem = (long)(LlapDaemon.getTotalHeapSize() * metadataFraction);
+      }
+      LowLevelCachePolicy cachePolicy = useLrfu ? new LowLevelLrfuCachePolicy(
+          minAllocSize, totalMemorySize, conf) : new LowLevelFifoCachePolicy();
       // Allocator uses memory manager to request memory, so create the manager next.
       LowLevelCacheMemoryManager memManager = new LowLevelCacheMemoryManager(
-          conf, cachePolicy, cacheMetrics);
+          totalMemorySize, cachePolicy, cacheMetrics);
+      LowLevelCachePolicy metaCachePolicy = null;
+      LowLevelCacheMemoryManager metaMemManager = null;
+      if (isSplitCache) {
+        metaCachePolicy = useLrfu ? new LowLevelLrfuCachePolicy(
+            minAllocSize, metaMem, conf) : new LowLevelFifoCachePolicy();
+        metaMemManager = new LowLevelCacheMemoryManager(metaMem, metaCachePolicy, cacheMetrics);
+      } else {
+        metaCachePolicy = cachePolicy;
+        metaMemManager = memManager;
+      }
+      cacheMetrics.setCacheCapacityTotal(totalMemorySize + metaMem);
       // Cache uses allocator to allocate and deallocate, create allocator and then caches.
       EvictionAwareAllocator allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
       this.allocator = allocator;
@@ -138,10 +160,13 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch>
{
         serdeCache = serdeCacheImpl;
       }
       boolean useGapCache = HiveConf.getBoolVar(conf, ConfVars.LLAP_CACHE_ENABLE_ORC_GAP_CACHE);
-      metadataCache = new OrcMetadataCache(memManager, cachePolicy, useGapCache);
+      metadataCache = new OrcMetadataCache(metaMemManager, metaCachePolicy, useGapCache);
       // And finally cache policy uses cache to notify it of eviction. The cycle is complete!
-      cachePolicy.setEvictionListener(new EvictionDispatcher(
-          cache, serdeCache, metadataCache, allocator));
+      EvictionDispatcher e = new EvictionDispatcher(cache, serdeCache, metadataCache, allocator);
+      if (isSplitCache) {
+        metaCachePolicy.setEvictionListener(e);
+      }
+      cachePolicy.setEvictionListener(e);
       cachePolicy.setParentDebugDumper(cacheImpl);
       cacheImpl.startThreads(); // Start the cache threads.
       bufferManager = cacheImpl; // Cache also serves as buffer manager.


Mime
View raw message