hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anasta...@apache.org
Subject hbase git commit: HBASE-19506: The CellChunkMap index chunks are usually small, so in order to prevent memory underutilization, HBASE-19506 presents small chunks preallocated in a small pool
Date Wed, 21 Feb 2018 08:12:42 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 79d9403a7 -> 92d04d575


HBASE-19506: The CellChunkMap index chunks are usually small, so in order to prevent memory underutilization, HBASE-19506 presents small chunks preallocated in a small pool


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/92d04d57
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/92d04d57
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/92d04d57

Branch: refs/heads/master
Commit: 92d04d57516391376d34aaca6307e5906ffa4ae8
Parents: 79d9403
Author: anastas <anastas@yahoo-inc.com>
Authored: Wed Feb 21 10:12:25 2018 +0200
Committer: anastas <anastas@yahoo-inc.com>
Committed: Wed Feb 21 10:12:25 2018 +0200

----------------------------------------------------------------------
 .../regionserver/CellChunkImmutableSegment.java |  76 +++--
 .../hadoop/hbase/regionserver/CellChunkMap.java |  17 +-
 .../apache/hadoop/hbase/regionserver/Chunk.java |   4 +
 .../hadoop/hbase/regionserver/ChunkCreator.java | 324 ++++++++++++++-----
 .../regionserver/ImmutableMemStoreLAB.java      |  28 +-
 .../hadoop/hbase/regionserver/MemStoreLAB.java  |  22 +-
 .../hbase/regionserver/MemStoreLABImpl.java     |  74 +++--
 .../regionserver/TestCompactingMemStore.java    |   3 +-
 .../regionserver/TestMemStoreChunkPool.java     |   6 +-
 .../hbase/regionserver/TestMemStoreLAB.java     |   8 +-
 .../TestMemstoreLABWithoutPool.java             |   2 +-
 11 files changed, 377 insertions(+), 187 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
index 53458f1..0eebfb5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.yetus.audience.InterfaceAudience;
 
 
-
 /**
  * CellChunkImmutableSegment extends the API supported by a {@link Segment},
  * and {@link ImmutableSegment}. This immutable segment is working with CellSet with
@@ -43,6 +42,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
 
   public static final long DEEP_OVERHEAD_CCM =
       ImmutableSegment.DEEP_OVERHEAD + ClassSize.CELL_CHUNK_MAP;
+  public static final float INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE = 0.1f;
 
   /////////////////////  CONSTRUCTORS  /////////////////////
   /**------------------------------------------------------------------------
@@ -135,20 +135,12 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
   private void initializeCellSet(int numOfCells, MemStoreSegmentsIterator iterator,
       MemStoreCompactionStrategy.Action action) {
 
-    // calculate how many chunks we will need for index
-    int chunkSize = ChunkCreator.getInstance().getChunkSize();
-    int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK;
-    int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk);
     int numOfCellsAfterCompaction = 0;
     int currentChunkIdx = 0;
     int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
     int numUniqueKeys=0;
     Cell prev = null;
-    // all index Chunks are allocated from ChunkCreator
-    Chunk[] chunks = new Chunk[numberOfChunks];
-    for (int i=0; i < numberOfChunks; i++) {
-      chunks[i] = this.getMemStoreLAB().getNewExternalChunk();
-    }
+    Chunk[] chunks = allocIndexChunks(numOfCells);
     while (iterator.hasNext()) {        // the iterator hides the elimination logic for compaction
       boolean alreadyCopied = false;
       Cell c = iterator.next();
@@ -161,7 +153,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
         c = copyCellIntoMSLAB(c);
         alreadyCopied = true;
       }
-      if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) {
+      if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
         currentChunkIdx++;              // continue to the next index chunk
         offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
       }
@@ -207,15 +199,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
       int numOfCells, KeyValueScanner segmentScanner, CellSet oldCellSet,
       MemStoreCompactionStrategy.Action action) {
     Cell curCell;
-    // calculate how many chunks we will need for metadata
-    int chunkSize = ChunkCreator.getInstance().getChunkSize();
-    int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK;
-    int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk);
-    // all index Chunks are allocated from ChunkCreator
-    Chunk[] chunks = new Chunk[numberOfChunks];
-    for (int i=0; i < numberOfChunks; i++) {
-      chunks[i] = this.getMemStoreLAB().getNewExternalChunk();
-    }
+    Chunk[] chunks = allocIndexChunks(numOfCells);
 
     int currentChunkIdx = 0;
     int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
@@ -231,7 +215,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
           // are copied into MSLAB here.
           curCell = copyCellIntoMSLAB(curCell);
         }
-        if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) {
+        if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
           // continue to the next metadata chunk
           currentChunkIdx++;
           offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
@@ -279,14 +263,58 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
     return offset;
   }
 
-  private int calculateNumberOfChunks(int numOfCells, int numOfCellsInChunk) {
-    int numberOfChunks = numOfCells/numOfCellsInChunk;
-    if(numOfCells%numOfCellsInChunk!=0) { // if cells cannot be divided evenly between chunks
+  private int calculateNumberOfChunks(int numOfCells, int chunkSize) {
+    int numOfCellsInChunk = calcNumOfCellsInChunk(chunkSize);
+    int numberOfChunks = numOfCells / numOfCellsInChunk;
+    if(numOfCells % numOfCellsInChunk != 0) { // if cells cannot be divided evenly between chunks
       numberOfChunks++;                   // add one additional chunk
     }
     return numberOfChunks;
   }
 
+  // Assuming we are going to use regular data chunks as index chunks,
+  // we check here how much free space will remain in the last allocated chunk
+  // (the least occupied one).
+  // If the percentage of its remaining free space is above the INDEX_CHUNK_UNUSED_SPACE
+  // threshold, then we will use index chunks (which are smaller) instead.
+  private ChunkCreator.ChunkType useIndexChunks(int numOfCells) {
+    int dataChunkSize = ChunkCreator.getInstance().getChunkSize();
+    int numOfCellsInChunk = calcNumOfCellsInChunk(dataChunkSize);
+    int cellsInLastChunk = numOfCells % numOfCellsInChunk;
+    if (cellsInLastChunk == 0) { // There is no free space in the last chunk and thus,
+      return ChunkCreator.ChunkType.DATA_CHUNK;               // no need to use index chunks.
+    } else {
+      int chunkSpace = dataChunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER;
+      int freeSpaceInLastChunk = chunkSpace - cellsInLastChunk * ClassSize.CELL_CHUNK_MAP_ENTRY;
+      if (freeSpaceInLastChunk > INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE * chunkSpace) {
+        return ChunkCreator.ChunkType.INDEX_CHUNK;
+      }
+      return ChunkCreator.ChunkType.DATA_CHUNK;
+    }
+  }
+
+  private int calcNumOfCellsInChunk(int chunkSize) {
+    int chunkSpace = chunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER;
+    int numOfCellsInChunk = chunkSpace / ClassSize.CELL_CHUNK_MAP_ENTRY;
+    return numOfCellsInChunk;
+  }
+
+  private Chunk[] allocIndexChunks(int numOfCells) {
+    // Decide whether to use regular or small chunks and then
+    // calculate how many chunks we will need for index
+
+    ChunkCreator.ChunkType chunkType = useIndexChunks(numOfCells);
+    int chunkSize = ChunkCreator.getInstance().getChunkSize(chunkType);
+    int numberOfChunks = calculateNumberOfChunks(numOfCells, chunkSize);
+    // all index Chunks are allocated from ChunkCreator
+    Chunk[] chunks = new Chunk[numberOfChunks];
+    // all index Chunks are allocated from ChunkCreator
+    for (int i = 0; i < numberOfChunks; i++) {
+      chunks[i] = this.getMemStoreLAB().getNewExternalChunk(chunkType);
+    }
+    return chunks;
+  }
+
   private Cell copyCellIntoMSLAB(Cell cell) {
     // Take care for a special case when a cell is copied from on-heap to (probably off-heap) MSLAB.
     // The cell allocated as an on-heap JVM object (byte array) occupies slightly different

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
index a3b45c3..a2c1820 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
@@ -57,11 +57,10 @@ public class CellChunkMap extends CellFlatMap {
 
   private final Chunk[] chunks;             // the array of chunks, on which the index is based
 
-  // constant number of cell-representations in a chunk
+  // number of cell-representations in a chunk
+  // depends on the size of the chunks (may be index chunks or regular data chunks)
   // each chunk starts with its own ID following the cells data
-  public static final int NUM_OF_CELL_REPS_IN_CHUNK =
-      (ChunkCreator.getInstance().getChunkSize() - ChunkCreator.SIZEOF_CHUNK_HEADER) /
-          ClassSize.CELL_CHUNK_MAP_ENTRY;
+  private final int numOfCellRepsInChunk;
 
   /**
    * C-tor for creating CellChunkMap from existing Chunk array, which must be ordered
@@ -76,6 +75,12 @@ public class CellChunkMap extends CellFlatMap {
       Chunk[] chunks, int min, int max, boolean descending) {
     super(comparator, min, max, descending);
     this.chunks = chunks;
+    if (chunks != null && chunks.length != 0 && chunks[0] != null) {
+      this.numOfCellRepsInChunk = (chunks[0].size - ChunkCreator.SIZEOF_CHUNK_HEADER) /
+              ClassSize.CELL_CHUNK_MAP_ENTRY;
+    } else { // In case the chunks array was not allocated
+      this.numOfCellRepsInChunk = 0;
+    }
   }
 
   /* To be used by base (CellFlatMap) class only to create a sub-CellFlatMap
@@ -89,9 +94,9 @@ public class CellChunkMap extends CellFlatMap {
   @Override
   protected Cell getCell(int i) {
     // get the index of the relevant chunk inside chunk array
-    int chunkIndex = (i / NUM_OF_CELL_REPS_IN_CHUNK);
+    int chunkIndex = (i / numOfCellRepsInChunk);
     ByteBuffer block = chunks[chunkIndex].getData();// get the ByteBuffer of the relevant chunk
-    int j = i - chunkIndex * NUM_OF_CELL_REPS_IN_CHUNK; // get the index of the cell-representation
+    int j = i - chunkIndex * numOfCellRepsInChunk; // get the index of the cell-representation
 
     // find inside the offset inside the chunk holding the index, skip bytes for chunk id
     int offsetInBytes = ChunkCreator.SIZEOF_CHUNK_HEADER + j* ClassSize.CELL_CHUNK_MAP_ENTRY;

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
index 199298b..136efee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
@@ -89,6 +89,10 @@ public abstract class Chunk {
     return size > ChunkCreator.getInstance().getChunkSize();
   }
 
+  boolean isIndexChunk() {
+    return size == ChunkCreator.getInstance().getChunkSize(ChunkCreator.ChunkType.INDEX_CHUNK);
+  }
+
   /**
    * Actually claim the memory for this chunk. This should only be called from the thread that
    * constructed the chunk. It is thread-safe against other threads calling alloc(), who will block

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
index b2d4ba8..e62303e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
@@ -56,27 +56,58 @@ public class ChunkCreator {
   // the header size need to be changed in case chunk id size is changed
   public static final int SIZEOF_CHUNK_HEADER = Bytes.SIZEOF_INT;
 
+  /**
+   * Types of chunks, based on their sizes
+   */
+  public enum ChunkType {
+    // An index chunk is a small chunk, allocated from the index chunks pool.
+    // Its size is fixed and is 10% of the size of a data chunk.
+    INDEX_CHUNK,
+    // A data chunk is a regular chunk, allocated from the data chunks pool.
+    // Its size is fixed and given as input to the ChunkCreator c'tor.
+    DATA_CHUNK,
+    // A jumbo chunk isn't allocated from pool. Its size is bigger than the size of a
+    // data chunk, and is determined per chunk (meaning, there is no fixed jumbo size).
+    JUMBO_CHUNK
+  }
+
   // mapping from chunk IDs to chunks
   private Map<Integer, Chunk> chunkIdMap = new ConcurrentHashMap<Integer, Chunk>();
 
-  private final int chunkSize;
   private final boolean offheap;
   @VisibleForTesting
-  static ChunkCreator INSTANCE;
+  static ChunkCreator instance;
   @VisibleForTesting
   static boolean chunkPoolDisabled = false;
-  private MemStoreChunkPool pool;
+  private MemStoreChunkPool dataChunksPool;
+  private int chunkSize;
+  private MemStoreChunkPool indexChunksPool;
 
   @VisibleForTesting
   ChunkCreator(int chunkSize, boolean offheap, long globalMemStoreSize, float poolSizePercentage,
-      float initialCountPercentage, HeapMemoryManager heapMemoryManager) {
-    this.chunkSize = chunkSize;
+               float initialCountPercentage, HeapMemoryManager heapMemoryManager,
+               float indexChunkSizePercentage) {
     this.offheap = offheap;
-    this.pool = initializePool(globalMemStoreSize, poolSizePercentage, initialCountPercentage);
-    if (heapMemoryManager != null && this.pool != null) {
-      // Register with Heap Memory manager
-      heapMemoryManager.registerTuneObserver(this.pool);
-    }
+    this.chunkSize = chunkSize; // in case pools are not allocated
+    initializePools(chunkSize, globalMemStoreSize, poolSizePercentage, indexChunkSizePercentage,
+            initialCountPercentage, heapMemoryManager);
+  }
+
+  @VisibleForTesting
+  private void initializePools(int chunkSize, long globalMemStoreSize,
+                               float poolSizePercentage, float indexChunkSizePercentage,
+                               float initialCountPercentage,
+                               HeapMemoryManager heapMemoryManager) {
+    this.dataChunksPool = initializePool(globalMemStoreSize,
+            (1 - indexChunkSizePercentage) * poolSizePercentage,
+            initialCountPercentage, chunkSize, heapMemoryManager);
+    // The index chunks pool is needed only when the index type is CCM.
+    // Since the pools are not created at all when the index type isn't CCM,
+    // we don't need to check it here.
+    this.indexChunksPool = initializePool(globalMemStoreSize,
+            indexChunkSizePercentage * poolSizePercentage,
+            initialCountPercentage, (int) (indexChunkSizePercentage * chunkSize),
+            heapMemoryManager);
   }
 
   /**
@@ -90,18 +121,30 @@ public class ChunkCreator {
    * @return singleton MSLABChunkCreator
    */
   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "LI_LAZY_INIT_STATIC",
-      justification = "Method is called by single thread at the starting of RS")
+          justification = "Method is called by single thread at the starting of RS")
   @VisibleForTesting
   public static ChunkCreator initialize(int chunkSize, boolean offheap, long globalMemStoreSize,
-      float poolSizePercentage, float initialCountPercentage, HeapMemoryManager heapMemoryManager) {
-    if (INSTANCE != null) return INSTANCE;
-    INSTANCE = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage,
-        initialCountPercentage, heapMemoryManager);
-    return INSTANCE;
+                                        float poolSizePercentage, float initialCountPercentage,
+                                        HeapMemoryManager heapMemoryManager) {
+    if (instance != null) {
+      return instance;
+    }
+    instance = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage,
+            initialCountPercentage, heapMemoryManager,
+            MemStoreLABImpl.INDEX_CHUNK_PERCENTAGE_DEFAULT);
+    return instance;
   }
 
   static ChunkCreator getInstance() {
-    return INSTANCE;
+    return instance;
+  }
+
+  /**
+   * Creates and inits a chunk. The default implementation for a specific chunk size.
+   * @return the chunk that was initialized
+   */
+  Chunk getChunk(ChunkType chunkType) {
+    return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkType);
   }
 
   /**
@@ -109,15 +152,37 @@ public class ChunkCreator {
    * @return the chunk that was initialized
    */
   Chunk getChunk() {
-    return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
+    return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, ChunkType.DATA_CHUNK);
   }
 
   /**
-   * Creates and inits a chunk. The default implementation for specific index type.
+   * Creates and inits a chunk. The default implementation for a specific index type.
    * @return the chunk that was initialized
    */
   Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) {
-    return getChunk(chunkIndexType, chunkSize);
+    return getChunk(chunkIndexType, ChunkType.DATA_CHUNK);
+  }
+
+  /**
+   * Creates and inits a chunk with specific index type and type.
+   * @return the chunk that was initialized
+   */
+  Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, ChunkType chunkType) {
+    switch (chunkType) {
+      case INDEX_CHUNK:
+        if (indexChunksPool != null) {
+          return getChunk(chunkIndexType, indexChunksPool.getChunkSize());
+        }
+      case DATA_CHUNK:
+        if (dataChunksPool == null) {
+          return getChunk(chunkIndexType, chunkSize);
+        } else {
+          return getChunk(chunkIndexType, dataChunksPool.getChunkSize());
+        }
+      default:
+        throw new IllegalArgumentException(
+                "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
+    }
   }
 
   /**
@@ -128,18 +193,28 @@ public class ChunkCreator {
    */
   Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, int size) {
     Chunk chunk = null;
-    // if we have pool and this is not jumbo chunk (when size != chunkSize this is jumbo chunk)
-    if ((pool != null) && (size == chunkSize)) {
+    MemStoreChunkPool pool = null;
+
+    // if the size is suitable for one of the pools
+    if (dataChunksPool != null && size == dataChunksPool.getChunkSize()) {
+      pool = dataChunksPool;
+    } else if (indexChunksPool != null && size == indexChunksPool.getChunkSize()) {
+      pool = indexChunksPool;
+    }
+
+    // if we have a pool
+    if (pool != null) {
       //  the pool creates the chunk internally. The chunk#init() call happens here
-      chunk = this.pool.getChunk();
+      chunk = pool.getChunk();
       // the pool has run out of maxCount
       if (chunk == null) {
         if (LOG.isTraceEnabled()) {
-          LOG.trace("Chunk pool full (maxCount={}); creating chunk offheap.",
-              this.pool.getMaxCount());
+          LOG.trace("The chunk pool is full. Reached maxCount= " + pool.getMaxCount()
+                  + ". Creating chunk onheap.");
         }
       }
     }
+
     if (chunk == null) {
       // the second parameter explains whether CellChunkMap index is requested,
       // in that case, put allocated on demand chunk mapping into chunkIdMap
@@ -156,18 +231,18 @@ public class ChunkCreator {
    * Creates and inits a chunk of a special size, bigger than a regular chunk size.
    * Such a chunk will never come from pool and will always be on demand allocated.
    * @return the chunk that was initialized
-   * @param chunkIndexType whether the requested chunk is going to be used with CellChunkMap index
    * @param jumboSize the special size to be used
    */
-  Chunk getJumboChunk(CompactingMemStore.IndexType chunkIndexType, int jumboSize) {
-    if (jumboSize <= chunkSize) {
-      LOG.warn("Jumbo chunk size=" + jumboSize + " must be more than regular chunk size="
-          + chunkSize + "; converting to regular chunk.");
-      return getChunk(chunkIndexType,chunkSize);
+  Chunk getJumboChunk(int jumboSize) {
+    int allocSize = jumboSize + SIZEOF_CHUNK_HEADER;
+    if (allocSize <= dataChunksPool.getChunkSize()) {
+      LOG.warn("Jumbo chunk size " + jumboSize + " must be more than regular chunk size "
+              + dataChunksPool.getChunkSize() + ". Converting to regular chunk.");
+      return getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
     }
-    // the size of the allocation includes
-    // both the size requested and a place for the Chunk's header
-    return getChunk(chunkIndexType, jumboSize + SIZEOF_CHUNK_HEADER);
+    // the new chunk is going to hold the jumbo cell data and needs to be referenced by
+    // a strong map. Therefore the CCM index type
+    return getChunk(CompactingMemStore.IndexType.CHUNK_MAP, allocSize);
   }
 
   /**
@@ -196,21 +271,21 @@ public class ChunkCreator {
 
   // Chunks from pool are created covered with strong references anyway
   // TODO: change to CHUNK_MAP if it is generally defined
-  private Chunk createChunkForPool() {
-    return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
+  private Chunk createChunkForPool(CompactingMemStore.IndexType chunkIndexType, int chunkSize) {
+    if (chunkSize != dataChunksPool.getChunkSize() &&
+            chunkSize != indexChunksPool.getChunkSize()) {
+      return null;
+    }
+    return createChunk(true, chunkIndexType, chunkSize);
   }
 
   @VisibleForTesting
-  // Used to translate the ChunkID into a chunk ref
+    // Used to translate the ChunkID into a chunk ref
   Chunk getChunk(int id) {
     // can return null if chunk was never mapped
     return chunkIdMap.get(id);
   }
 
-  int getChunkSize() {
-    return this.chunkSize;
-  }
-
   boolean isOffheap() {
     return this.offheap;
   }
@@ -224,8 +299,8 @@ public class ChunkCreator {
   }
 
   @VisibleForTesting
-  // the chunks in the chunkIdMap may already be released so we shouldn't relay
-  // on this counting for strong correctness. This method is used only in testing.
+    // the chunks in the chunkIdMap may already be released so we shouldn't relay
+    // on this counting for strong correctness. This method is used only in testing.
   int numberOfMappedChunks() {
     return this.chunkIdMap.size();
   }
@@ -243,6 +318,7 @@ public class ChunkCreator {
    * collection on JVM.
    */
   private  class MemStoreChunkPool implements HeapMemoryTuneObserver {
+    private final int chunkSize;
     private int maxCount;
 
     // A queue of reclaimed chunks
@@ -256,21 +332,22 @@ public class ChunkCreator {
     private final AtomicLong chunkCount = new AtomicLong();
     private final LongAdder reusedChunkCount = new LongAdder();
 
-    MemStoreChunkPool(int maxCount, int initialCount, float poolSizePercentage) {
+    MemStoreChunkPool(int chunkSize, int maxCount, int initialCount, float poolSizePercentage) {
+      this.chunkSize = chunkSize;
       this.maxCount = maxCount;
       this.poolSizePercentage = poolSizePercentage;
       this.reclaimedChunks = new LinkedBlockingQueue<>();
       for (int i = 0; i < initialCount; i++) {
-        Chunk chunk = createChunkForPool();
+        Chunk chunk = createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
         chunk.init();
         reclaimedChunks.add(chunk);
       }
       chunkCount.set(initialCount);
       final String n = Thread.currentThread().getName();
       scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
-          .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build());
+              .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build());
       this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(), statThreadPeriod,
-          statThreadPeriod, TimeUnit.SECONDS);
+              statThreadPeriod, TimeUnit.SECONDS);
     }
 
     /**
@@ -283,6 +360,10 @@ public class ChunkCreator {
      * @see #putbackChunks(Chunk)
      */
     Chunk getChunk() {
+      return getChunk(CompactingMemStore.IndexType.ARRAY_MAP);
+    }
+
+    Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) {
       Chunk chunk = reclaimedChunks.poll();
       if (chunk != null) {
         chunk.reset();
@@ -293,7 +374,7 @@ public class ChunkCreator {
           long created = this.chunkCount.get();
           if (created < this.maxCount) {
             if (this.chunkCount.compareAndSet(created, created + 1)) {
-              chunk = createChunkForPool();
+              chunk = createChunkForPool(chunkIndexType, chunkSize);
               break;
             }
           } else {
@@ -304,6 +385,10 @@ public class ChunkCreator {
       return chunk;
     }
 
+    int getChunkSize() {
+      return chunkSize;
+    }
+
     /**
      * Add the chunks to the pool, when the pool achieves the max size, it will skip the remaining
      * chunks
@@ -311,7 +396,7 @@ public class ChunkCreator {
      */
     private void putbackChunks(Chunk c) {
       int toAdd = this.maxCount - reclaimedChunks.size();
-      if (c.isFromPool() && toAdd > 0) {
+      if (c.isFromPool() && c.size == chunkSize && toAdd > 0) {
         reclaimedChunks.add(c);
       } else {
         // remove the chunk (that is not going to pool)
@@ -336,10 +421,11 @@ public class ChunkCreator {
         long created = chunkCount.get();
         long reused = reusedChunkCount.sum();
         long total = created + reused;
-        LOG.debug("Stats: current pool size=" + reclaimedChunks.size()
-            + ",created chunk count=" + created
-            + ",reused chunk count=" + reused
-            + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent(
+        LOG.debug("Stats (chunk size=" + chunkSize + "): "
+                + "current pool size=" + reclaimedChunks.size()
+                + ",created chunk count=" + created
+                + ",reused chunk count=" + reused
+                + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent(
                 (float) reused / (float) total, 2)));
       }
     }
@@ -356,7 +442,7 @@ public class ChunkCreator {
         return;
       }
       int newMaxCount =
-          (int) (newMemstoreSize * poolSizePercentage / getChunkSize());
+              (int) (newMemstoreSize * poolSizePercentage / getChunkSize());
       if (newMaxCount != this.maxCount) {
         // We need an adjustment in the chunks numbers
         if (newMaxCount > this.maxCount) {
@@ -387,7 +473,8 @@ public class ChunkCreator {
   }
 
   private MemStoreChunkPool initializePool(long globalMemStoreSize, float poolSizePercentage,
-      float initialCountPercentage) {
+                                           float initialCountPercentage, int chunkSize,
+                                           HeapMemoryManager heapMemoryManager) {
     if (poolSizePercentage <= 0) {
       LOG.info("PoolSizePercentage is less than 0. So not using pool");
       return null;
@@ -397,47 +484,90 @@ public class ChunkCreator {
     }
     if (poolSizePercentage > 1.0) {
       throw new IllegalArgumentException(
-          MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
+              MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
     }
-    int maxCount = (int) (globalMemStoreSize * poolSizePercentage / getChunkSize());
+    int maxCount = (int) (globalMemStoreSize * poolSizePercentage / chunkSize);
     if (initialCountPercentage > 1.0 || initialCountPercentage < 0) {
       throw new IllegalArgumentException(
-          MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0");
+              MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0");
     }
     int initialCount = (int) (initialCountPercentage * maxCount);
-    LOG.info("Allocating MemStoreChunkPool with chunk size="
-        + StringUtils.byteDesc(getChunkSize()) + ", max count=" + maxCount
-        + ", initial count=" + initialCount);
-    return new MemStoreChunkPool(maxCount, initialCount, poolSizePercentage);
+    LOG.info("Allocating MemStoreChunkPool with chunk size "
+            + StringUtils.byteDesc(chunkSize) + ", max count " + maxCount
+            + ", initial count " + initialCount);
+    MemStoreChunkPool memStoreChunkPool = new MemStoreChunkPool(chunkSize, maxCount,
+            initialCount, poolSizePercentage);
+    if (heapMemoryManager != null && memStoreChunkPool != null) {
+      // Register with Heap Memory manager
+      heapMemoryManager.registerTuneObserver(memStoreChunkPool);
+    }
+    return memStoreChunkPool;
   }
 
   @VisibleForTesting
   int getMaxCount() {
-    if (pool != null) {
-      return pool.getMaxCount();
+    return getMaxCount(ChunkType.DATA_CHUNK);
+  }
+
+  @VisibleForTesting
+  int getMaxCount(ChunkType chunkType) {
+    switch (chunkType) {
+      case INDEX_CHUNK:
+        if (indexChunksPool != null) {
+          return indexChunksPool.getMaxCount();
+        }
+        break;
+      case DATA_CHUNK:
+        if (dataChunksPool != null) {
+          return dataChunksPool.getMaxCount();
+        }
+        break;
+      default:
+        throw new IllegalArgumentException(
+                "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
     }
+
     return 0;
   }
 
   @VisibleForTesting
   int getPoolSize() {
-    if (pool != null) {
-      return pool.reclaimedChunks.size();
+    return getPoolSize(ChunkType.DATA_CHUNK);
+  }
+
+  @VisibleForTesting
+  int getPoolSize(ChunkType chunkType) {
+    switch (chunkType) {
+      case INDEX_CHUNK:
+        if (indexChunksPool != null) {
+          return indexChunksPool.reclaimedChunks.size();
+        }
+        break;
+      case DATA_CHUNK:
+        if (dataChunksPool != null) {
+          return dataChunksPool.reclaimedChunks.size();
+        }
+        break;
+      default:
+        throw new IllegalArgumentException(
+                "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
     }
     return 0;
   }
 
   @VisibleForTesting
   boolean isChunkInPool(int chunkId) {
-    if (pool != null) {
-      // chunks that are from pool will return true chunk reference not null
-      Chunk c = getChunk(chunkId);
-      if (c==null) {
-        return false;
-      }
-      return pool.reclaimedChunks.contains(c);
+    Chunk c = getChunk(chunkId);
+    if (c==null) {
+      return false;
     }
 
+    // chunks that are from pool will return true chunk reference not null
+    if (dataChunksPool != null && dataChunksPool.reclaimedChunks.contains(c)) {
+      return true;
+    } else if (indexChunksPool != null && indexChunksPool.reclaimedChunks.contains(c)) {
+      return true;
+    }
     return false;
   }
 
@@ -446,31 +576,58 @@ public class ChunkCreator {
    */
   @VisibleForTesting
   void clearChunksInPool() {
-    if (pool != null) {
-      pool.reclaimedChunks.clear();
+    if (dataChunksPool != null) {
+      dataChunksPool.reclaimedChunks.clear();
+    }
+    if (indexChunksPool != null) {
+      indexChunksPool.reclaimedChunks.clear();
+    }
+  }
+
+  int getChunkSize() {
+    return getChunkSize(ChunkType.DATA_CHUNK);
+  }
+
+  int getChunkSize(ChunkType chunkType) {
+    switch (chunkType) {
+      case INDEX_CHUNK:
+        if (indexChunksPool != null) {
+          return indexChunksPool.getChunkSize();
+        }
+      case DATA_CHUNK:
+        if (dataChunksPool != null) {
+          return dataChunksPool.getChunkSize();
+        } else { // When pools are empty
+          return chunkSize;
+        }
+      default:
+        throw new IllegalArgumentException(
+                "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
     }
   }
 
   synchronized void putbackChunks(Set<Integer> chunks) {
     // if there is no pool just try to clear the chunkIdMap in case there is something
-    if ( pool == null ) {
+    if (dataChunksPool == null && indexChunksPool == null) {
       this.removeChunks(chunks);
       return;
     }
 
-    // if there is pool, go over all chunk IDs that came back, the chunks may be from pool or not
+    // if there is a pool, go over all chunk IDs that came back, the chunks may be from pool or not
     for (int chunkID : chunks) {
       // translate chunk ID to chunk, if chunk initially wasn't in pool
       // this translation will (most likely) return null
       Chunk chunk = ChunkCreator.this.getChunk(chunkID);
       if (chunk != null) {
-        // Jumbo chunks are covered with chunkIdMap, but are not from pool, so such a chunk should
-        // be released here without going to pool.
-        // Removing them from chunkIdMap will cause their removal by the GC.
-        if (chunk.isJumbo()) {
-          this.removeChunk(chunkID);
+        if (chunk.isFromPool() && chunk.isIndexChunk()) {
+          indexChunksPool.putbackChunks(chunk);
+        } else if (chunk.isFromPool() && chunk.size == dataChunksPool.getChunkSize()) {
+          dataChunksPool.putbackChunks(chunk);
         } else {
-          pool.putbackChunks(chunk);
+          // chunks which are not from one of the pools
+          // should be released without going to the pools.
+          // Removing them from chunkIdMap will cause their removal by the GC.
+          this.removeChunk(chunkID);
         }
       }
       // if chunk is null, it was never covered by the chunkIdMap (and so wasn't in pool also),
@@ -480,3 +637,4 @@ public class ChunkCreator {
   }
 
 }
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
index 71648a0..0ce52b6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
@@ -61,32 +61,26 @@ public class ImmutableMemStoreLAB implements MemStoreLAB {
     return mslab.forceCopyOfBigCellInto(cell);
   }
 
-  /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
-  ** Returning a new chunk, without replacing current chunk,
+  /* Returning a new pool chunk, without replacing current chunk,
   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
   ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
+  ** The interface is only for external callers.
   */
   @Override
-  public Chunk getNewExternalChunk() {
+  public Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType) {
     MemStoreLAB mslab = this.mslabs.get(0);
-    return mslab.getNewExternalChunk();
+    return mslab.getNewExternalChunk(chunkType);
   }
 
-  /* Creating chunk to be used as data chunk in CellChunkMap.
-  ** This chunk is bigger the normal constant chunk size, and thus called JumboChunk it is used for
-  ** jumbo cells (which size is bigger than normal chunks).
-  ** Jumbo Chunks are needed only for CCM and thus are created only in
-  ** CompactingMemStore.IndexType.CHUNK_MAP type.
-  ** Returning a new chunk, without replacing current chunk,
-  ** meaning MSLABImpl does not make the returned chunk as CurChunk.
-  ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
-  */
+  /* Returning a new chunk, without replacing current chunk,
+   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
+   ** The space on this chunk will be allocated externally.
+   ** The interface is only for external callers.
+   */
   @Override
-  public Chunk getNewExternalJumboChunk(int size) {
+  public Chunk getNewExternalChunk(int size) {
     MemStoreLAB mslab = this.mslabs.get(0);
-    return mslab.getNewExternalJumboChunk(size);
+    return mslab.getNewExternalChunk(size);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
index 8b77981..90cf932 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
@@ -52,6 +52,8 @@ public interface MemStoreLAB {
 
   String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize";
   int CHUNK_SIZE_DEFAULT = 2048 * 1024;
+  String INDEX_CHUNK_PERCENTAGE_KEY = "hbase.hregion.memstore.mslab.indexchunksize";
+  float INDEX_CHUNK_PERCENTAGE_DEFAULT = 0.1f;
   String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation";
   int MAX_ALLOC_DEFAULT = 256 * 1024; // allocs bigger than this don't go through
                                                    // allocator
@@ -94,25 +96,19 @@ public interface MemStoreLAB {
    */
   void decScannerCount();
 
-  /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
-  ** Returning a new chunk, without replacing current chunk,
+  /* Returning a new pool chunk, without replacing current chunk,
   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
   ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
+  ** The interface is only for external callers.
   */
-  Chunk getNewExternalChunk();
-
-  /* Creating chunk to be used as data chunk in CellChunkMap.
-  ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk it is used for
-  ** jumbo cells (which size is bigger than normal chunks).
-  ** Jumbo Chunks are needed only for CCM and thus are created only in
-  ** CompactingMemStore.IndexType.CHUNK_MAP type.
-  ** Returning a new chunk, without replacing current chunk,
+  Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType);
+
+  /* Returning a new chunk, without replacing current chunk,
   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
   ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
+  ** The interface is only for external callers.
   */
-  Chunk getNewExternalJumboChunk(int size);
+  Chunk getNewExternalChunk(int size);
 
   static MemStoreLAB newInstance(Configuration conf) {
     MemStoreLAB memStoreLAB = null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
index 4ff0480..0db0fd9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
@@ -67,14 +67,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
 
   static final Logger LOG = LoggerFactory.getLogger(MemStoreLABImpl.class);
 
-  private AtomicReference<Chunk> curChunk = new AtomicReference<>();
+  private AtomicReference<Chunk> currChunk = new AtomicReference<>();
   // Lock to manage multiple handlers requesting for a chunk
   private ReentrantLock lock = new ReentrantLock();
 
   // A set of chunks contained by this memstore LAB
   @VisibleForTesting
   Set<Integer> chunks = new ConcurrentSkipListSet<Integer>();
-  private final int chunkSize;
+  private final int dataChunkSize;
   private final int maxAlloc;
   private final ChunkCreator chunkCreator;
   private final CompactingMemStore.IndexType idxType; // what index is used for corresponding segment
@@ -94,11 +94,11 @@ public class MemStoreLABImpl implements MemStoreLAB {
   }
 
   public MemStoreLABImpl(Configuration conf) {
-    chunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT);
+    dataChunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT);
     maxAlloc = conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT);
     this.chunkCreator = ChunkCreator.getInstance();
     // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one!
-    Preconditions.checkArgument(maxAlloc <= chunkSize,
+    Preconditions.checkArgument(maxAlloc <= dataChunkSize,
         MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY);
 
     // if user requested to work with MSLABs (whether on- or off-heap), then the
@@ -120,14 +120,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
    */
   @Override
   public Cell forceCopyOfBigCellInto(Cell cell) {
-    int size = KeyValueUtil.length(cell);
+    int size = KeyValueUtil.length(cell) + ChunkCreator.SIZEOF_CHUNK_HEADER;
     Preconditions.checkArgument(size >= 0, "negative size");
-    if (size <= chunkSize) {
+    if (size <= dataChunkSize) {
       // Using copyCellInto for cells which are bigger than the original maxAlloc
-      Cell newCell = copyCellInto(cell, chunkSize);
+      Cell newCell = copyCellInto(cell, dataChunkSize);
       return newCell;
     } else {
-      Chunk c = getNewExternalJumboChunk(size);
+      Chunk c = getNewExternalChunk(size);
       int allocOffset = c.alloc(size);
       return copyToChunkCell(cell, c.getData(), allocOffset, size);
     }
@@ -240,7 +240,7 @@ public class MemStoreLABImpl implements MemStoreLAB {
    * @return true if we won the race to retire the chunk
    */
   private void tryRetireChunk(Chunk c) {
-    curChunk.compareAndSet(c, null);
+    currChunk.compareAndSet(c, null);
     // If the CAS succeeds, that means that we won the race
     // to retire the chunk. We could use this opportunity to
     // update metrics on external fragmentation.
@@ -255,7 +255,8 @@ public class MemStoreLABImpl implements MemStoreLAB {
    */
   private Chunk getOrMakeChunk() {
     // Try to get the chunk
-    Chunk c = curChunk.get();
+    Chunk c;
+    c = currChunk.get();
     if (c != null) {
       return c;
     }
@@ -265,14 +266,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
     if (lock.tryLock()) {
       try {
         // once again check inside the lock
-        c = curChunk.get();
+        c = currChunk.get();
         if (c != null) {
           return c;
         }
         c = this.chunkCreator.getChunk(idxType);
         if (c != null) {
           // set the curChunk. No need of CAS as only one thread will be here
-          curChunk.set(c);
+          currChunk.set(c);
           chunks.add(c.getId());
           return c;
         }
@@ -283,38 +284,41 @@ public class MemStoreLABImpl implements MemStoreLAB {
     return null;
   }
 
-  /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
-  ** Returning a new chunk, without replacing current chunk,
+  /* Returning a new pool chunk, without replacing current chunk,
   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
   ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
+  ** The interface is only for external callers.
   */
   @Override
-  public Chunk getNewExternalChunk() {
-    // the new chunk is going to be part of the chunk array and will always be referenced
-    Chunk c = this.chunkCreator.getChunk();
-    chunks.add(c.getId());
-    return c;
+  public Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType) {
+    switch (chunkType) {
+      case INDEX_CHUNK:
+      case DATA_CHUNK:
+        Chunk c = this.chunkCreator.getChunk(chunkType);
+        chunks.add(c.getId());
+        return c;
+      case JUMBO_CHUNK: // a jumbo chunk doesn't have a fixed size
+      default:
+        return null;
+    }
   }
 
-  /* Creating chunk to be used as data chunk in CellChunkMap.
-  ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk.
-  ** JumboChunk is used for jumbo cell (which size is bigger than normal chunk). It is allocated
-  ** once per cell. So even if there is space this is not reused.
-  ** Jumbo Chunks are used only for CCM and thus are created only in
-  ** CompactingMemStore.IndexType.CHUNK_MAP type.
-  ** Returning a new chunk, without replacing current chunk,
+  /* Returning a new chunk, without replacing current chunk,
   ** meaning MSLABImpl does not make the returned chunk as CurChunk.
   ** The space on this chunk will be allocated externally.
-  ** The interface is only for external callers
+  ** The interface is only for external callers.
+  ** Chunks from pools are not allocated from here, since they have fixed sizes
   */
   @Override
-  public Chunk getNewExternalJumboChunk(int size) {
-    // the new chunk is going to hold the jumbo cell data and need to be referenced by a strong map
-    // thus giving the CCM index type
-    Chunk c = this.chunkCreator.getJumboChunk(CompactingMemStore.IndexType.CHUNK_MAP, size);
-    chunks.add(c.getId());
-    return c;
+  public Chunk getNewExternalChunk(int size) {
+    int allocSize = size + ChunkCreator.getInstance().SIZEOF_CHUNK_HEADER;
+    if (allocSize <= ChunkCreator.getInstance().getChunkSize()) {
+      return getNewExternalChunk(ChunkCreator.ChunkType.DATA_CHUNK);
+    } else {
+      Chunk c = this.chunkCreator.getJumboChunk(size);
+      chunks.add(c.getId());
+      return c;
+    }
   }
 
   @Override
@@ -329,7 +333,7 @@ public class MemStoreLABImpl implements MemStoreLAB {
 
   @VisibleForTesting
   Chunk getCurrentChunk() {
-    return this.curChunk.get();
+    return currChunk.get();
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index 505c2f0..5cbfff9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -118,7 +118,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
     long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
         .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
     chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
-        globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
+        globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT,
+            null);
     assertTrue(chunkCreator != null);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
index 891e835..4f3de36 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
@@ -233,10 +233,10 @@ public class TestMemStoreChunkPool {
     final int chunkSize = 40;
     final int valSize = 7;
     ChunkCreator oldCreator = ChunkCreator.getInstance();
-    ChunkCreator newCreator = new ChunkCreator(chunkSize, false, 400, 1, 0.5f, null);
+    ChunkCreator newCreator = new ChunkCreator(chunkSize, false, 400, 1, 0.5f, null, 0);
     assertEquals(initialCount, newCreator.getPoolSize());
     assertEquals(maxCount, newCreator.getMaxCount());
-    ChunkCreator.INSTANCE = newCreator;// Replace the global ref with the new one we created.
+    ChunkCreator.instance = newCreator;// Replace the global ref with the new one we created.
                                              // Used it for the testing. Later in finally we put
                                              // back the original
     final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"),
@@ -265,7 +265,7 @@ public class TestMemStoreChunkPool {
       t3.join();
       assertTrue(newCreator.getPoolSize() <= maxCount);
     } finally {
-      ChunkCreator.INSTANCE = oldCreator;
+      ChunkCreator.instance = oldCreator;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
index 52491e7..ef4ad69 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
@@ -206,8 +206,8 @@ public class TestMemStoreLAB {
       MemStoreLABImpl mslab = new MemStoreLABImpl();
       // by default setting, there should be no chunks initialized in the pool
       assertTrue(mslab.getPooledChunks().isEmpty());
-      oldInstance = ChunkCreator.INSTANCE;
-      ChunkCreator.INSTANCE = null;
+      oldInstance = ChunkCreator.instance;
+      ChunkCreator.instance = null;
       // reset mslab with chunk pool
       Configuration conf = HBaseConfiguration.create();
       conf.setDouble(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.1);
@@ -251,7 +251,7 @@ public class TestMemStoreLAB {
       }
       // none of the chunkIds would have been returned back
       assertTrue("All the chunks must have been cleared",
-          ChunkCreator.INSTANCE.numberOfMappedChunks() != 0);
+          ChunkCreator.instance.numberOfMappedChunks() != 0);
       int pooledChunksNum = mslab.getPooledChunks().size();
       // close the mslab
       mslab.close();
@@ -261,7 +261,7 @@ public class TestMemStoreLAB {
           + " after mslab closed but actually: " + (pooledChunksNum-queueLength),
           pooledChunksNum-queueLength == 0);
     } finally {
-      ChunkCreator.INSTANCE = oldInstance;
+      ChunkCreator.instance = oldInstance;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
index 0813a8c..e2da5d0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
@@ -148,7 +148,7 @@ public class TestMemstoreLABWithoutPool {
     }
     // all of the chunkIds would have been returned back
     assertTrue("All the chunks must have been cleared",
-        ChunkCreator.INSTANCE.numberOfMappedChunks() == 0);
+        ChunkCreator.instance.numberOfMappedChunks() == 0);
   }
 
   private Thread getChunkQueueTestThread(final MemStoreLABImpl mslab, String threadName,


Mime
View raw message