hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject hbase git commit: HBASE-20523 PE tool should support configuring client side buffering sizes (Ram)
Date Mon, 07 May 2018 07:30:17 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 5e14e125b -> 8e6ff689e


HBASE-20523 PE tool should support configuring client side buffering sizes
(Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e6ff689
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e6ff689
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e6ff689

Branch: refs/heads/master
Commit: 8e6ff689e806138e602a46730886d8c5f524fdcd
Parents: 5e14e12
Author: Vasudevan <ramkrishna.s.vasudevan@intel.com>
Authored: Mon May 7 12:50:24 2018 +0530
Committer: Vasudevan <ramkrishna.s.vasudevan@intel.com>
Committed: Mon May 7 12:57:20 2018 +0530

----------------------------------------------------------------------
 .../hadoop/hbase/PerformanceEvaluation.java     | 23 +++++++++++++++++++-
 .../hadoop/hbase/TestPerformanceEvaluation.java | 10 +++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8e6ff689/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 7fc064f..42dc984 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.Append;
 import org.apache.hadoop.hbase.client.AsyncConnection;
 import org.apache.hadoop.hbase.client.AsyncTable;
 import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.BufferedMutatorParams;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Consistency;
@@ -666,6 +667,7 @@ public class PerformanceEvaluation extends Configured implements Tool
{
     boolean asyncPrefetch = false;
     boolean cacheBlocks = true;
     Scan.ReadType scanReadType = Scan.ReadType.DEFAULT;
+    long bufferSize = 2l * 1024l * 1024l;
 
     public TestOptions() {}
 
@@ -715,6 +717,7 @@ public class PerformanceEvaluation extends Configured implements Tool
{
       this.asyncPrefetch = that.asyncPrefetch;
       this.cacheBlocks = that.cacheBlocks;
       this.scanReadType = that.scanReadType;
+      this.bufferSize = that.bufferSize;
     }
 
     public int getCaching() {
@@ -885,6 +888,10 @@ public class PerformanceEvaluation extends Configured implements Tool
{
       this.valueSize = valueSize;
     }
 
+    public void setBufferSize(long bufferSize) {
+      this.bufferSize = bufferSize;
+    }
+
     public void setPeriod(int period) {
       this.period = period;
     }
@@ -1020,6 +1027,10 @@ public class PerformanceEvaluation extends Configured implements Tool
{
     public MemoryCompactionPolicy getInMemoryCompaction() {
       return this.inMemoryCompaction;
     }
+
+    public long getBufferSize() {
+      return this.bufferSize;
+    }
   }
 
   /*
@@ -1626,7 +1637,9 @@ public class PerformanceEvaluation extends Configured implements Tool
{
 
     @Override
     void onStartup() throws IOException {
-      this.mutator = connection.getBufferedMutator(TableName.valueOf(opts.tableName));
+      BufferedMutatorParams p = new BufferedMutatorParams(TableName.valueOf(opts.tableName));
+      p.writeBufferSize(opts.bufferSize);
+      this.mutator = connection.getBufferedMutator(p);
       this.table = connection.getTable(TableName.valueOf(opts.tableName));
     }
 
@@ -2363,6 +2376,7 @@ public class PerformanceEvaluation extends Configured implements Tool
{
     System.err.println(" asyncPrefetch   Enable asyncPrefetch for scan");
     System.err.println(" cacheBlocks     Set the cacheBlocks option for scan. Default: true");
     System.err.println(" scanReadType    Set the readType option for scan, stream/pread/default.
Default: default");
+    System.err.println(" bufferSize      Set the value of client side buffering. Default:
2MB");
     System.err.println();
     System.err.println(" Note: -D properties will be applied to the conf used. ");
     System.err.println("  For example: ");
@@ -2636,6 +2650,13 @@ public class PerformanceEvaluation extends Configured implements Tool
{
             Scan.ReadType.valueOf(cmd.substring(scanReadType.length()).toUpperCase());
         continue;
       }
+
+      final String bufferSize = "--bufferSize=";
+      if (cmd.startsWith(bufferSize)) {
+        opts.bufferSize = Long.parseLong(cmd.substring(bufferSize.length()));
+        continue;
+      }
+
       if (isCommandClass(cmd)) {
         opts.cmdName = cmd;
         try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/8e6ff689/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index d5b25e2..ab93857 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -179,6 +179,16 @@ public class TestPerformanceEvaluation {
   }
 
   @Test
+  public void testSetBufferSizeOption() {
+    TestOptions opts = new PerformanceEvaluation.TestOptions();
+    long bufferSize = opts.getBufferSize();
+    assertEquals(bufferSize, 2l * 1024l * 1024l);
+    opts.setBufferSize(64l * 1024l);
+    bufferSize = opts.getBufferSize();
+    assertEquals(bufferSize, 64l * 1024l);
+  }
+
+  @Test
   public void testParseOptsWithThreads() {
     Queue<String> opts = new LinkedList<>();
     String cmdName = "sequentialWrite";


Mime
View raw message