From commits-return-22020-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Thu Aug 2 19:37:38 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 35CDB180674 for ; Thu, 2 Aug 2018 19:37:38 +0200 (CEST) Received: (qmail 97846 invoked by uid 500); 2 Aug 2018 17:37:37 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 97753 invoked by uid 99); 2 Aug 2018 17:37:37 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 02 Aug 2018 17:37:37 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id C67F8825A8; Thu, 2 Aug 2018 17:37:36 +0000 (UTC) Date: Thu, 02 Aug 2018 17:37:36 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo-testing] branch master updated: Use hints (apache/accumulo#555) in scan executor test (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <153323145676.29982.6738264720091588569@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo-testing X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: c2eb3f28687aeb0ec6ee2273340b8e68281374f8 X-Git-Newrev: 6aff9ce2295d6658775f93539967b785c155d604 X-Git-Rev: 6aff9ce2295d6658775f93539967b785c155d604 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/master by this push: new 6aff9ce Use hints (apache/accumulo#555) in scan executor test (#31) 6aff9ce is described below commit 6aff9ce2295d6658775f93539967b785c155d604 Author: Keith Turner AuthorDate: Thu Aug 2 13:37:35 2018 -0400 Use hints (apache/accumulo#555) in scan executor test (#31) --- .../core/performance/tests/ScanExecutorPT.java | 63 ++++++++++++++++------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java index ecfa182..3dbd5da 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java @@ -34,7 +34,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.spi.scan.IdleRatioScanPrioritizer; +import org.apache.accumulo.core.spi.scan.HintScanPrioritizer; import org.apache.accumulo.testing.core.performance.Environment; import org.apache.accumulo.testing.core.performance.PerformanceTest; import org.apache.accumulo.testing.core.performance.Report; @@ -43,6 +43,7 @@ import org.apache.accumulo.testing.core.performance.util.TestData; import org.apache.accumulo.testing.core.performance.util.TestExecutor; import org.apache.hadoop.io.Text; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; public class ScanExecutorPT implements PerformanceTest { @@ -55,10 +56,13 @@ public class ScanExecutorPT implements PerformanceTest { private static final int NUM_QUALS = 10; private static final String SCAN_EXECUTOR_THREADS = "2"; - private static final String SCAN_PRIORITIZER = IdleRatioScanPrioritizer.class.getName(); + private static final String SCAN_PRIORITIZER = HintScanPrioritizer.class.getName(); - private static final String TEST_DESC = "Scan Executor Test. Test running lots of short scans while long scans are running in the background. Each short scan reads a random row and family. A scan prioritizer that favors short scans is configured. If the scan prioritizer is not working properly, then the short " - + "scans will be orders of magnitude slower."; + private static final String TEST_DESC = "Scan Executor Test. Test running lots of short scans " + + "while long scans are running in the background. Each short scan reads a random row and " + + "family. Using execution hints, short scans are randomly either given a high priority or " + + "a dedicated executor. If the scan prioritizer or dispatcher is not working properly, " + + "then the short scans will be orders of magnitude slower."; @Override public SystemConfiguration getConfiguration() { @@ -66,8 +70,14 @@ public class ScanExecutorPT implements PerformanceTest { siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "200"); siteCfg.put(Property.TSERV_MINTHREADS.getKey(), "200"); - siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.threads", SCAN_EXECUTOR_THREADS); - siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.prioritizer", SCAN_PRIORITIZER); + siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.threads", + SCAN_EXECUTOR_THREADS); + siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.prioritizer", + SCAN_PRIORITIZER); + siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se2.threads", + SCAN_EXECUTOR_THREADS); + siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se2.prioritizer", + SCAN_PRIORITIZER); return new SystemConfiguration().setAccumuloConfig(siteCfg); } @@ -79,8 +89,10 @@ public class ScanExecutorPT implements PerformanceTest { Map props = new HashMap<>(); props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "executor", "se1"); + props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "heed_hints", "true"); - env.getConnector().tableOperations().create(tableName, new NewTableConfiguration().setProperties(props)); + env.getConnector().tableOperations().create(tableName, + new NewTableConfiguration().setProperties(props)); long t1 = System.currentTimeMillis(); TestData.generate(env.getConnector(), tableName, NUM_ROWS, NUM_FAMS, NUM_QUALS); @@ -109,24 +121,30 @@ public class ScanExecutorPT implements PerformanceTest { builder.info("compact", NUM_ROWS * NUM_FAMS * NUM_QUALS, t3 - t2, "Compact rate entries/sec "); builder.info("short_times1", shortStats1, "Times in ms for each short scan. First run."); builder.info("short_times2", shortStats2, "Times in ms for each short scan. Second run."); - builder.result("short", shortStats2.getAverage(), "Average times in ms for short scans from 2nd run."); + builder.result("short", shortStats2.getAverage(), + "Average times in ms for short scans from 2nd run."); builder.info("long_counts", longStats, "Entries read by each long scan threads"); - builder.info("long", longStats.getSum(), (t4-t3), "Combined rate in entries/second of all long scans"); + builder.info("long", longStats.getSum(), (t4 - t3), + "Combined rate in entries/second of all long scans"); builder.parameter("short_threads", NUM_SHORT_SCANS_THREADS, "Threads used to run short scans."); - builder.parameter("long_threads", NUM_LONG_SCANS, "Threads running long scans. Each thread repeatedly scans entire table for duration of test."); + builder.parameter("long_threads", NUM_LONG_SCANS, + "Threads running long scans. Each thread repeatedly scans entire table for duration of test."); builder.parameter("rows", NUM_ROWS, "Rows in test table"); builder.parameter("familes", NUM_FAMS, "Families per row in test table"); builder.parameter("qualifiers", NUM_QUALS, "Qualifiers per family in test table"); - builder.parameter("server_scan_threads", SCAN_EXECUTOR_THREADS, "Server side scan handler threads"); + builder.parameter("server_scan_threads", SCAN_EXECUTOR_THREADS, + "Server side scan handler threads"); builder.parameter("prioritizer", SCAN_PRIORITIZER, "Server side scan prioritizer"); return builder.build(); } - private static long scan(String tableName, Connector c, byte[] row, byte[] fam) throws TableNotFoundException { + private static long scan(String tableName, Connector c, byte[] row, byte[] fam, + Map hints) throws TableNotFoundException { long t1 = System.currentTimeMillis(); int count = 0; try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) { + scanner.setExecutionHints(hints); scanner.setRange(Range.exact(new Text(row), new Text(fam))); if (Iterables.size(scanner) != NUM_QUALS) { throw new RuntimeException("bad count " + count); @@ -136,10 +154,12 @@ public class ScanExecutorPT implements PerformanceTest { return System.currentTimeMillis() - t1; } - private long scan(String tableName, Connector c, AtomicBoolean stop) throws TableNotFoundException { + private long scan(String tableName, Connector c, AtomicBoolean stop, Map hints) + throws TableNotFoundException { long count = 0; while (!stop.get()) { try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) { + scanner.setExecutionHints(hints); for (Entry entry : scanner) { count++; if (stop.get()) { @@ -151,15 +171,22 @@ public class ScanExecutorPT implements PerformanceTest { return count; } - private LongSummaryStatistics runShortScans(Environment env, String tableName, int numScans) throws InterruptedException, ExecutionException { + private LongSummaryStatistics runShortScans(Environment env, String tableName, int numScans) + throws InterruptedException, ExecutionException { - try(TestExecutor executor = new TestExecutor<>(NUM_SHORT_SCANS_THREADS)) { + Map execHints = ImmutableMap.of("executor", "se2"); + Map prioHints = ImmutableMap.of("priority", "1"); + + try (TestExecutor executor = new TestExecutor<>(NUM_SHORT_SCANS_THREADS)) { Random rand = new Random(); for (int i = 0; i < numScans; i++) { byte[] row = TestData.row(rand.nextInt(NUM_ROWS)); byte[] fam = TestData.fam(rand.nextInt(NUM_FAMS)); - executor.submit(() -> scan(tableName, env.getConnector(), row, fam)); + // scans have a 20% chance of getting dedicated thread pool and 80% chance of getting high + // priority + Map hints = rand.nextInt(10) <= 1 ? execHints : prioHints; + executor.submit(() -> scan(tableName, env.getConnector(), row, fam, hints)); } return executor.stream().mapToLong(l -> l).summaryStatistics(); @@ -167,10 +194,12 @@ public class ScanExecutorPT implements PerformanceTest { } private TestExecutor startLongScans(Environment env, String tableName, AtomicBoolean stop) { + Map hints = ImmutableMap.of("priority", "2"); + TestExecutor longScans = new TestExecutor<>(NUM_LONG_SCANS); for (int i = 0; i < NUM_LONG_SCANS; i++) { - longScans.submit(() -> scan(tableName, env.getConnector(), stop)); + longScans.submit(() -> scan(tableName, env.getConnector(), stop, hints)); } return longScans; }