hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [16/17] hbase git commit: HBASE-14721 Memstore add cells - Avoid many garbage.
Date Fri, 30 Oct 2015 16:54:15 GMT
HBASE-14721 Memstore add cells - Avoid many garbage.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23fa1818
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23fa1818
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23fa1818

Branch: refs/heads/hbase-12439
Commit: 23fa18184cb68ca05246beb2189f8801200bdd7c
Parents: 84f4cce
Author: anoopsjohn <anoopsamjohn@gmail.com>
Authored: Fri Oct 30 18:39:42 2015 +0530
Committer: anoopsjohn <anoopsamjohn@gmail.com>
Committed: Fri Oct 30 18:39:42 2015 +0530

----------------------------------------------------------------------
 .../hbase/regionserver/DefaultMemStore.java     | 18 ++++++------------
 .../hadoop/hbase/regionserver/HRegion.java      | 20 ++++++--------------
 .../hadoop/hbase/regionserver/HStore.java       |  3 +--
 .../hadoop/hbase/regionserver/MemStore.java     |  6 ++----
 .../apache/hadoop/hbase/regionserver/Store.java |  5 ++---
 .../hadoop/hbase/regionserver/TestStore.java    | 17 ++++++-----------
 6 files changed, 23 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 9bc6a9c..d760e9a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.CollectionBackedScanner;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.htrace.Trace;
 
@@ -221,13 +220,12 @@ public class DefaultMemStore implements MemStore {
   /**
    * Write an update
    * @param cell
-   * @return approximate size of the passed KV &amp; newly added KV which maybe different
than the
-   *         passed-in KV
+   * @return approximate size of the passed Cell.
    */
   @Override
-  public Pair<Long, Cell> add(Cell cell) {
+  public long add(Cell cell) {
     Cell toAdd = maybeCloneWithAllocator(cell);
-    return new Pair<Long, Cell>(internalAdd(toAdd), toAdd);
+    return internalAdd(toAdd);
   }
 
   @Override
@@ -964,21 +962,17 @@ public class DefaultMemStore implements MemStore {
     byte [] empty = new byte[0];
     for (int i = 0; i < count; i++) {
       // Give each its own ts
-      Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf,
i, empty));
-      size += ret.getFirst();
+      size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
     }
     LOG.info("memstore1 estimated size=" + size);
     for (int i = 0; i < count; i++) {
-      Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf,
i, empty));
-      size += ret.getFirst();
+      size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
     }
     LOG.info("memstore1 estimated size (2nd loading of same data)=" + size);
     // Make a variably sized memstore.
     DefaultMemStore memstore2 = new DefaultMemStore();
     for (int i = 0; i < count; i++) {
-      Pair<Long, Cell> ret = memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf,
i,
-        new byte[i]));
-      size += ret.getFirst();
+      size += memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]));
     }
     LOG.info("memstore2 estimated size=" + size);
     final int seconds = 30;

http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 19bcd33..e987bc6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3686,15 +3686,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
       int listSize = cells.size();
       for (int i=0; i < listSize; i++) {
         Cell cell = cells.get(i);
-        if (cell.getSequenceId() == 0) {
+        if (cell.getSequenceId() == 0 || isInReplay) {
           CellUtil.setSequenceId(cell, mvccNum);
         }
-        Pair<Long, Cell> ret = store.add(cell);
-        size += ret.getFirst();
-        if(isInReplay) {
-          // set memstore newly added cells with replay mvcc number
-          CellUtil.setSequenceId(ret.getSecond(), mvccNum);
-        }
+        size += store.add(cell);
       }
     }
 
@@ -4937,7 +4932,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
    * @return True if we should flush.
    */
   protected boolean restoreEdit(final Store s, final Cell cell) {
-    long kvSize = s.add(cell).getFirst();
+    long kvSize = s.add(cell);
     if (this.rsAccounting != null) {
       rsAccounting.addAndGetRegionReplayEditsSize(getRegionInfo().getRegionName(), kvSize);
     }
@@ -6829,8 +6824,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
                 checkFamily(CellUtil.cloneFamily(cell));
                 // unreachable
               }
-              Pair<Long, Cell> ret = store.add(cell);
-              addedSize += ret.getFirst();
+              addedSize += store.add(cell);
             }
           }
 
@@ -7168,8 +7162,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
                 // otherwise keep older versions around
                 for (Cell cell: entry.getValue()) {
                   CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
-                  Pair<Long, Cell> ret = store.add(cell);
-                  size += ret.getFirst();
+                  size += store.add(cell);
                   doRollBackMemstore = true;
                 }
               }
@@ -7399,8 +7392,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
                 // otherwise keep older versions around
                 for (Cell cell : entry.getValue()) {
                   CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
-                  Pair<Long, Cell> ret = store.add(cell);
-                  size += ret.getFirst();
+                  size += store.add(cell);
                   doRollBackMemstore = true;
                 }
               }

http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index c0f1d55..1d996b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.StringUtils;
@@ -675,7 +674,7 @@ public class HStore implements Store {
   }
 
   @Override
-  public Pair<Long, Cell> add(final Cell cell) {
+  public long add(final Cell cell) {
     lock.readLock().lock();
     try {
        return this.memstore.add(cell);

http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index d24299d..e9f8103 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -22,7 +22,6 @@ import java.util.List;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.util.Pair;
 
 /**
  * The MemStore holds in-memory modifications to the Store. Modifications are {@link Cell}s.
@@ -68,10 +67,9 @@ public interface MemStore extends HeapSize {
   /**
    * Write an update
    * @param cell
-   * @return approximate size of the passed KV and the newly added KV which maybe different
from the
-   *         passed in KV.
+   * @return approximate size of the passed cell.
    */
-  Pair<Long, Cell> add(final Cell cell);
+  long add(final Cell cell);
 
   /**
    * @return Oldest timestamp of all the Cells in the MemStore

http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 33e4416..83a24a5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
 import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.Pair;
 
 /**
  * Interface for objects that hold a column family in a Region. Its a memstore and a set
of zero or
@@ -127,9 +126,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
   /**
    * Adds a value to the memstore
    * @param cell
-   * @return memstore size delta &amp; newly added KV which maybe different than the
passed in KV
+   * @return memstore size delta
    */
-  Pair<Long, Cell> add(Cell cell);
+  long add(Cell cell);
 
   /**
    * When was the last edit done in the memstore

http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 6b669a0..85c75b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -223,7 +223,7 @@ public class TestStore {
         long size = store.memstore.getFlushableSize();
         Assert.assertEquals(0, size);
         LOG.info("Adding some data");
-        long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null)).getFirst();
+        long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
         size = store.memstore.getFlushableSize();
         Assert.assertEquals(kvSize, size);
         // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is
right.
@@ -638,20 +638,15 @@ public class TestStore {
 
 
     size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
-        System.currentTimeMillis(),
-        Bytes.toBytes(newValue))).getFirst();
+        System.currentTimeMillis(), Bytes.toBytes(newValue)));
     size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
-        System.currentTimeMillis(),
-        Bytes.toBytes(newValue))).getFirst();
+        System.currentTimeMillis(), Bytes.toBytes(newValue)));
     size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
-        System.currentTimeMillis(),
-        Bytes.toBytes(newValue))).getFirst();
+        System.currentTimeMillis(), Bytes.toBytes(newValue)));
     size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
-        System.currentTimeMillis(),
-        Bytes.toBytes(newValue))).getFirst();
+        System.currentTimeMillis(), Bytes.toBytes(newValue)));
     size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
-        System.currentTimeMillis(),
-        Bytes.toBytes(newValue))).getFirst();
+        System.currentTimeMillis(), Bytes.toBytes(newValue)));
 
 
     for ( int i = 0 ; i < 10000 ; ++i) {


Mime
View raw message