Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CD12E11D63 for ; Tue, 24 Jun 2014 09:34:43 +0000 (UTC) Received: (qmail 20244 invoked by uid 500); 24 Jun 2014 09:34:43 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 20199 invoked by uid 500); 24 Jun 2014 09:34:43 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 20184 invoked by uid 99); 24 Jun 2014 09:34:43 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Jun 2014 09:34:43 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 44F4A90F9C2; Tue, 24 Jun 2014 09:34:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nkeywal@apache.org To: commits@hbase.apache.org Message-Id: <0ea9083163f54f3c82eddbc1b5263cf1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HBASE-11390 PerformanceEvaluation: add an option to use a single connection Date: Tue, 24 Jun 2014 09:34:43 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/master b2914be9d -> e991fd805 HBASE-11390 PerformanceEvaluation: add an option to use a single connection Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e991fd80 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e991fd80 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e991fd80 Branch: refs/heads/master Commit: e991fd8055112e2ddc0bbeade2ef74504e2407dd Parents: b2914be Author: Nicolas Liochon Authored: Tue Jun 24 11:34:45 2014 +0200 Committer: Nicolas Liochon Committed: Tue Jun 24 11:34:45 2014 +0200 ---------------------------------------------------------------------- .../hadoop/hbase/PerformanceEvaluation.java | 83 ++++++++++++-------- 1 file changed, 50 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e991fd80/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index d54cfa1..5190fb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -503,6 +504,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.flushCommits = that.flushCommits; this.writeToWAL = that.writeToWAL; this.autoFlush = that.autoFlush; + this.oneCon = that.oneCon; this.useTags = that.useTags; this.noOfTags = that.noOfTags; this.reportLatency = that.reportLatency; @@ -531,6 +533,7 @@ public class PerformanceEvaluation extends Configured implements Tool { public boolean flushCommits = true; public boolean writeToWAL = true; public boolean autoFlush = false; + public boolean oneCon = false; public boolean useTags = false; public int noOfTags = 1; public boolean reportLatency = false; @@ -577,8 +580,9 @@ public class PerformanceEvaluation extends Configured implements Tool { * Note that all subclasses of this class must provide a public contructor * that has the exact same list of arguments. */ - Test(final Configuration conf, final TestOptions options, final Status status) { - this.conf = conf; + Test(final HConnection con, final TestOptions options, final Status status) { + this.connection = con; + this.conf = con.getConfiguration(); this.opts = options; this.status = status; this.testName = this.getClass().getSimpleName(); @@ -631,9 +635,11 @@ public class PerformanceEvaluation extends Configured implements Tool { } void testSetup() throws IOException { - this.connection = HConnectionManager.createConnection(conf); - this.table = connection.getTable(opts.tableName); - this.table.setAutoFlush(opts.autoFlush, true); + if (!opts.oneCon) { + this.connection = HConnectionManager.createConnection(conf); + } + this.table = new HTable(TableName.valueOf(opts.tableName), connection); + this.table.setAutoFlushTo(opts.autoFlush); try { Constructor ctor = @@ -654,7 +660,9 @@ public class PerformanceEvaluation extends Configured implements Tool { this.table.flushCommits(); } table.close(); - connection.close(); + if (!opts.oneCon) { + connection.close(); + } receiverHost.closeReceivers(); } @@ -765,8 +773,8 @@ public class PerformanceEvaluation extends Configured implements Tool { @SuppressWarnings("unused") static class RandomSeekScanTest extends Test { - RandomSeekScanTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomSeekScanTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -795,8 +803,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static abstract class RandomScanWithRangeTest extends Test { - RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomScanWithRangeTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -811,7 +819,7 @@ public class PerformanceEvaluation extends Configured implements Tool { int count = 0; ResultScanner s = this.table.getScanner(scan); for (; (r = s.next()) != null;) { - updateValueSize(r);; + updateValueSize(r); count++; } if (i % 100 == 0) { @@ -839,8 +847,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class RandomScanWithRange10Test extends RandomScanWithRangeTest { - RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomScanWithRange10Test(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -850,8 +858,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class RandomScanWithRange100Test extends RandomScanWithRangeTest { - RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomScanWithRange100Test(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -861,8 +869,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class RandomScanWithRange1000Test extends RandomScanWithRangeTest { - RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomScanWithRange1000Test(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -872,8 +880,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class RandomScanWithRange10000Test extends RandomScanWithRangeTest { - RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomScanWithRange10000Test(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -885,8 +893,8 @@ public class PerformanceEvaluation extends Configured implements Tool { static class RandomReadTest extends Test { private ArrayList gets; - RandomReadTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomReadTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); if (opts.multiGet > 0) { LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + "."); this.gets = new ArrayList(opts.multiGet); @@ -930,8 +938,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class RandomWriteTest extends Test { - RandomWriteTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + RandomWriteTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -962,8 +970,8 @@ public class PerformanceEvaluation extends Configured implements Tool { static class ScanTest extends Test { private ResultScanner testScanner; - ScanTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + ScanTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -993,8 +1001,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class SequentialReadTest extends Test { - SequentialReadTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + SequentialReadTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -1009,8 +1017,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } static class SequentialWriteTest extends Test { - SequentialWriteTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + SequentialWriteTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -1041,8 +1049,8 @@ public class PerformanceEvaluation extends Configured implements Tool { static class FilteredScanTest extends Test { protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName()); - FilteredScanTest(Configuration conf, TestOptions options, Status status) { - super(conf, options, status); + FilteredScanTest(HConnection con, TestOptions options, Status status) { + super(con, options, status); } @Override @@ -1159,10 +1167,11 @@ public class PerformanceEvaluation extends Configured implements Tool { long totalElapsedTime; final Test t; + HConnection con = HConnectionManager.createConnection(conf); try { Constructor constructor = - cmd.getDeclaredConstructor(Configuration.class, TestOptions.class, Status.class); - t = constructor.newInstance(conf, opts, status); + cmd.getDeclaredConstructor(HConnection.class, TestOptions.class, Status.class); + t = constructor.newInstance(con, opts, status); } catch (NoSuchMethodException e) { throw new IllegalArgumentException("Invalid command class: " + cmd.getName() + ". It does not provide a constructor as described by " + @@ -1177,6 +1186,7 @@ public class PerformanceEvaluation extends Configured implements Tool { "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts)) + ")"); + con.close(); return totalElapsedTime; } @@ -1229,6 +1239,7 @@ public class PerformanceEvaluation extends Configured implements Tool { "Default: false"); System.err.println(" writeToWAL Set writeToWAL on puts. Default: True"); System.err.println(" autoFlush Set autoFlush on htable. Default: False"); + System.err.println(" oneCon all the threads share the same connection. Default: False"); System.err.println(" presplit Create presplit table. Recommended for accurate perf " + "analysis (see guide). Default: disabled"); System.err.println(" inmemory Tries to keep the HFiles of the CF " + @@ -1366,6 +1377,12 @@ public class PerformanceEvaluation extends Configured implements Tool { continue; } + final String onceCon = "--oneCon="; + if (cmd.startsWith(onceCon)) { + opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length())); + continue; + } + final String presplit = "--presplit="; if (cmd.startsWith(presplit)) { opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));