hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-11156 : LLAP: Cache arena size rounding (Sergey Shelukhin)
Date Wed, 01 Jul 2015 23:52:19 GMT
Repository: hive
Updated Branches:
  refs/heads/llap 5dc48d596 -> 4339fd86c


HIVE-11156 : LLAP: Cache arena size rounding (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4339fd86
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4339fd86
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4339fd86

Branch: refs/heads/llap
Commit: 4339fd86cc243be0c4cc5580a496785c9b0753af
Parents: 5dc48d5
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Jul 1 16:52:10 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Jul 1 16:52:10 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hive/llap/cache/BuddyAllocator.java  | 20 ++++++++++++++------
 .../llap/cache/LowLevelCacheMemoryManager.java  |  8 ++++++--
 .../hadoop/hive/llap/cache/MemoryManager.java   |  1 +
 .../hive/llap/cache/TestBuddyAllocator.java     |  4 ++++
 .../hive/llap/cache/TestOrcMetadataCache.java   |  4 ++++
 5 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index 9f1472f..3631418 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -47,19 +47,19 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
     minAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_MIN_ALLOC);
     maxAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_ALLOC);
     arenaSize = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_ARENA_SIZE);
-    maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_SIZE);
+    long maxSizeVal = HiveConf.getLongVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_SIZE);
     if (LlapIoImpl.LOGL.isInfoEnabled()) {
       LlapIoImpl.LOG.info("Buddy allocator with " + (isDirect ? "direct" : "byte")
           + " buffers; allocation sizes " + minAllocation + " - " + maxAllocation
-          + ", arena size " + arenaSize + ". total size " + maxSize);
+          + ", arena size " + arenaSize + ". total size " + maxSizeVal);
     }
 
     if (minAllocation < 8) {
       throw new AssertionError("Min allocation must be at least 8: " + minAllocation);
     }
-    if (maxSize < arenaSize || arenaSize < maxAllocation || maxAllocation < minAllocation)
{
+    if (maxSizeVal < arenaSize || arenaSize < maxAllocation || maxAllocation < minAllocation)
{
       throw new AssertionError("Inconsistent sizes of cache, arena and allocations: "
-          + minAllocation + ", " + maxAllocation + ", " + arenaSize + ", " + maxSize);
+          + minAllocation + ", " + maxAllocation + ", " + arenaSize + ", " + maxSizeVal);
     }
     if ((Integer.bitCount(minAllocation) != 1) || (Integer.bitCount(maxAllocation) != 1)
         || (Long.bitCount(arenaSize) != 1)) {
@@ -67,10 +67,18 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
       throw new AssertionError("Allocation and arena sizes must be powers of two: "
           + minAllocation + ", " + maxAllocation + ", " + arenaSize);
     }
-    if ((maxSize % arenaSize) > 0 || (maxSize / arenaSize) > Integer.MAX_VALUE) {
+    if ((maxSizeVal % arenaSize) > 0) {
+      long oldMaxSize = maxSizeVal;
+      maxSizeVal = (maxSizeVal / arenaSize) * arenaSize;
+      LlapIoImpl.LOG.warn("Rounding cache size to " + maxSizeVal + " from " + oldMaxSize
+          + " to be divisible by arena size " + arenaSize);
+    }
+    if ((maxSizeVal / arenaSize) > Integer.MAX_VALUE) {
       throw new AssertionError(
-          "Cache size not consistent with arena size: " + arenaSize + "," + maxSize);
+          "Too many arenas needed to allocate the cache: " + arenaSize + "," + maxSizeVal);
     }
+    maxSize = maxSizeVal;
+    memoryManager.updateMaxSize(maxSize);
     minAllocLog2 = 31 - Integer.numberOfLeadingZeros(minAllocation);
     maxAllocLog2 = 31 - Integer.numberOfLeadingZeros(maxAllocation);
     arenaSizeLog2 = 63 - Long.numberOfLeadingZeros(arenaSize);

http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 85f66f8..4a256ee 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
  */
 public class LowLevelCacheMemoryManager implements MemoryManager {
   private final AtomicLong usedMemory;
-  protected final long maxSize;
   private final LowLevelCachePolicy evictor;
-  private LlapDaemonCacheMetrics metrics;
+  private final LlapDaemonCacheMetrics metrics;
+  private long maxSize;
 
   public LowLevelCacheMemoryManager(Configuration conf, LowLevelCachePolicy evictor,
       LlapDaemonCacheMetrics metrics) {
@@ -104,4 +104,8 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
     return "cache state\n" + evictor.debugDumpForOom();
   }
 
+  @Override
+  public void updateMaxSize(long maxSize) {
+    this.maxSize = maxSize;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
index 8e167ec..e1b0cb4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
@@ -21,4 +21,5 @@ package org.apache.hadoop.hive.llap.cache;
 public interface MemoryManager extends LlapOomDebugDump {
   boolean reserveMemory(long memoryToReserve, boolean waitForEviction);
   void releaseMemory(long memUsage);
+  void updateMaxSize(long maxSize);
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index d35edb7..7d265c1 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -54,6 +54,10 @@ public class TestBuddyAllocator {
     public String debugDumpForOom() {
       return "";
     }
+
+    @Override
+    public void updateMaxSize(long maxSize) {
+    }
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index 513aedf..3549fd9 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -74,6 +74,10 @@ public class TestOrcMetadataCache {
     public String debugDumpForOom() {
       return "";
     }
+
+    @Override
+    public void updateMaxSize(long maxSize) {
+    }
   }
 
   @Test


Mime
View raw message