From commits-return-22652-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Wed Feb 27 15:20:38 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B047518076D for ; Wed, 27 Feb 2019 16:20:36 +0100 (CET) Received: (qmail 31189 invoked by uid 500); 27 Feb 2019 15:20:35 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 31090 invoked by uid 99); 27 Feb 2019 15:20:35 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Feb 2019 15:20:35 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 09E3082E8A; Wed, 27 Feb 2019 15:20:35 +0000 (UTC) Date: Wed, 27 Feb 2019 15:20:34 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch master updated: Remove BatchWriterOpts and ScannerOpts (#992) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155128083462.32516.1345137950754688040@gitbox.apache.org> From: mwalch@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: d70a29fd5161c21bb268264070533fd05a607fb6 X-Git-Newrev: 4c015acf58fc6524429f3db815b86d2d56a52faf X-Git-Rev: 4c015acf58fc6524429f3db815b86d2d56a52faf X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. mwalch pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/master by this push: new 4c015ac Remove BatchWriterOpts and ScannerOpts (#992) 4c015ac is described below commit 4c015acf58fc6524429f3db815b86d2d56a52faf Author: Mike Walch AuthorDate: Wed Feb 27 10:20:30 2019 -0500 Remove BatchWriterOpts and ScannerOpts (#992) * These options can now be configured when AccumuloClient is created --- .../apache/accumulo/core/cli/BatchWriterOpts.java | 55 ---------------------- .../org/apache/accumulo/core/cli/ScannerOpts.java | 25 ---------- .../apache/accumulo/server/util/RandomWriter.java | 12 ++--- .../server/util/RemoveEntriesForMissingFiles.java | 6 +-- .../java/org/apache/accumulo/tracer/TraceDump.java | 14 ++---- .../org/apache/accumulo/test/TestBinaryRows.java | 44 +++++++---------- .../java/org/apache/accumulo/test/TestIngest.java | 18 ++++--- .../apache/accumulo/test/TestMultiTableIngest.java | 15 ++---- .../apache/accumulo/test/TestRandomDeletes.java | 29 +++++------- .../org/apache/accumulo/test/VerifyIngest.java | 9 ++-- .../BalanceInPresenceOfOfflineTableIT.java | 6 +-- .../apache/accumulo/test/functional/BinaryIT.java | 16 +++---- .../apache/accumulo/test/functional/BulkIT.java | 12 ++--- .../test/functional/BulkSplitOptimizationIT.java | 3 +- .../test/functional/ChaoticBalancerIT.java | 6 +-- .../accumulo/test/functional/CompactionIT.java | 3 +- .../apache/accumulo/test/functional/DeleteIT.java | 9 ++-- .../test/functional/DynamicThreadPoolsIT.java | 3 +- .../accumulo/test/functional/FateStarvationIT.java | 3 +- .../test/functional/FunctionalTestUtils.java | 14 ++---- .../test/functional/GarbageCollectorIT.java | 13 ++--- .../test/functional/HalfDeadTServerIT.java | 3 +- .../accumulo/test/functional/MasterFailoverIT.java | 6 +-- .../apache/accumulo/test/functional/MaxOpenIT.java | 3 +- .../accumulo/test/functional/ReadWriteIT.java | 7 +-- .../apache/accumulo/test/functional/RenameIT.java | 12 ++--- .../apache/accumulo/test/functional/RestartIT.java | 24 ++++------ .../accumulo/test/functional/RestartStressIT.java | 4 +- .../test/functional/SimpleBalancerFairnessIT.java | 3 +- .../apache/accumulo/test/functional/SplitIT.java | 6 +-- .../apache/accumulo/test/functional/TableIT.java | 10 ++-- .../accumulo/test/functional/WriteAheadLogIT.java | 8 ++-- .../accumulo/test/functional/WriteLotsIT.java | 38 +++++++-------- .../test/performance/ContinuousIngest.java | 7 +-- .../test/performance/RollWALPerformanceIT.java | 8 ++-- .../test/performance/scan/CollectTabletStats.java | 17 +++---- .../performance/scan/CollectTabletStatsTest.java | 7 +-- 37 files changed, 146 insertions(+), 332 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/cli/BatchWriterOpts.java b/core/src/main/java/org/apache/accumulo/core/cli/BatchWriterOpts.java deleted file mode 100644 index 96e4e85..0000000 --- a/core/src/main/java/org/apache/accumulo/core/cli/BatchWriterOpts.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.cli; - -import java.util.concurrent.TimeUnit; - -import org.apache.accumulo.core.cli.ClientOpts.MemoryConverter; -import org.apache.accumulo.core.cli.ClientOpts.TimeConverter; -import org.apache.accumulo.core.client.BatchWriterConfig; - -import com.beust.jcommander.Parameter; - -public class BatchWriterOpts { - private static final BatchWriterConfig BWDEFAULTS = new BatchWriterConfig(); - - @Parameter(names = "--batchThreads", - description = "Number of threads to use when writing large batches") - public Integer batchThreads = BWDEFAULTS.getMaxWriteThreads(); - - @Parameter(names = "--batchLatency", converter = TimeConverter.class, - description = "The maximum time to wait before flushing data to servers when writing") - public Long batchLatency = BWDEFAULTS.getMaxLatency(TimeUnit.MILLISECONDS); - - @Parameter(names = "--batchMemory", converter = MemoryConverter.class, - description = "memory used to batch data when writing") - public Long batchMemory = BWDEFAULTS.getMaxMemory(); - - @Parameter(names = "--batchTimeout", converter = TimeConverter.class, - description = "timeout used to fail a batch write") - public Long batchTimeout = BWDEFAULTS.getTimeout(TimeUnit.MILLISECONDS); - - public BatchWriterConfig getBatchWriterConfig() { - BatchWriterConfig config = new BatchWriterConfig(); - config.setMaxWriteThreads(this.batchThreads); - config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); - config.setMaxMemory(this.batchMemory); - config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); - return config; - } - -} diff --git a/core/src/main/java/org/apache/accumulo/core/cli/ScannerOpts.java b/core/src/main/java/org/apache/accumulo/core/cli/ScannerOpts.java deleted file mode 100644 index 37939fd..0000000 --- a/core/src/main/java/org/apache/accumulo/core/cli/ScannerOpts.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.cli; - -import com.beust.jcommander.Parameter; - -public class ScannerOpts { - @Parameter(names = "--scanBatchSize", - description = "the number of key-values to pull during a scan") - public int scanBatchSize = 1000; -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java b/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java index 56f60aa..89d42b5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomWriter.java @@ -20,7 +20,6 @@ import java.security.SecureRandom; import java.util.Iterator; import java.util.Random; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.cli.ClientOnDefaultTable; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; @@ -96,18 +95,13 @@ public class RandomWriter { public static void main(String[] args) throws Exception { Opts opts = new Opts(table_name); opts.setPrincipal("root"); - BatchWriterOpts bwOpts = new BatchWriterOpts(); - try (TraceScope clientSpan = opts.parseArgsAndTrace(RandomWriter.class.getName(), args, - bwOpts)) { - + try (TraceScope clientSpan = opts.parseArgsAndTrace(RandomWriter.class.getName(), args)) { long start = System.currentTimeMillis(); log.info("starting at {} for user {}", start, opts.getPrincipal()); - try (AccumuloClient accumuloClient = opts.createClient()) { - BatchWriter bw = accumuloClient.createBatchWriter(opts.getTableName(), - bwOpts.getBatchWriterConfig()); + try (AccumuloClient accumuloClient = opts.createClient(); + BatchWriter bw = accumuloClient.createBatchWriter(opts.getTableName())) { log.info("Writing {} mutations...", opts.count); bw.addMutations(new RandomMutationGenerator(opts.count)); - bw.close(); } catch (Exception e) { log.error("{}", e.getMessage(), e); throw e; diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java index ebc5113..64aee69 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java @@ -25,8 +25,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; @@ -204,10 +202,8 @@ public class RemoveEntriesForMissingFiles { public static void main(String[] args) throws Exception { Opts opts = new Opts(); - ScannerOpts scanOpts = new ScannerOpts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); try (TraceScope clientSpan = opts - .parseArgsAndTrace(RemoveEntriesForMissingFiles.class.getName(), args, scanOpts, bwOpts)) { + .parseArgsAndTrace(RemoveEntriesForMissingFiles.class.getName(), args)) { checkAllTables(opts.getServerContext(), opts.fix); } } diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceDump.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceDump.java index 76e1a7e..396345d 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceDump.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceDump.java @@ -28,7 +28,6 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.accumulo.core.cli.ClientOnDefaultTable; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.data.Key; @@ -59,14 +58,13 @@ public class TraceDump { public static void main(String[] args) throws Exception { Opts opts = new Opts(); - ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(TraceDump.class.getName(), args, scanOpts); + opts.parseArgs(TraceDump.class.getName(), args); int code = 0; if (opts.list) { - code = listSpans(opts, scanOpts); + code = listSpans(opts); } if (code == 0 && opts.dump) { - code = dumpTrace(opts, scanOpts); + code = dumpTrace(opts); } System.exit(code); } @@ -77,13 +75,12 @@ public class TraceDump { return spanList; } - private static int listSpans(Opts opts, ScannerOpts scanOpts) throws Exception { + private static int listSpans(Opts opts) throws Exception { PrintStream out = System.out; long endTime = System.currentTimeMillis(); long startTime = endTime - opts.length; try (AccumuloClient client = opts.createClient()) { Scanner scanner = client.createScanner(opts.getTableName(), opts.auths); - scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text("start:" + Long.toHexString(startTime)), new Text("start:" + Long.toHexString(endTime))); scanner.setRange(range); @@ -102,13 +99,12 @@ public class TraceDump { void print(String line); } - private static int dumpTrace(Opts opts, ScannerOpts scanOpts) throws Exception { + private static int dumpTrace(Opts opts) throws Exception { final PrintStream out = System.out; int count = 0; try (AccumuloClient client = opts.createClient()) { for (String traceId : opts.traceIds) { Scanner scanner = client.createScanner(opts.getTableName(), opts.auths); - scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text(traceId)); scanner.setRange(range); count = printTrace(scanner, out::println); diff --git a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java index 0de6992..da71cdd 100644 --- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java +++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java @@ -24,8 +24,6 @@ import java.util.Map.Entry; import java.util.Random; import java.util.TreeSet; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Scanner; @@ -87,33 +85,29 @@ public class TestBinaryRows { public long num = 0; } - public static void runTest(AccumuloClient accumuloClient, Opts opts, BatchWriterOpts bwOpts, - ScannerOpts scanOpts) throws Exception { + public static void runTest(AccumuloClient accumuloClient, Opts opts) throws Exception { final Text CF = new Text("cf"), CQ = new Text("cq"); final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8); if (opts.mode.equals("ingest") || opts.mode.equals("delete")) { - BatchWriter bw = accumuloClient.createBatchWriter(opts.getTableName(), - bwOpts.getBatchWriterConfig()); - boolean delete = opts.mode.equals("delete"); - - for (long i = 0; i < opts.num; i++) { - byte[] row = encodeLong(i + opts.start); - String value = "" + (i + opts.start); - - Mutation m = new Mutation(new Text(row)); - if (delete) { - m.putDelete(CF, CQ); - } else { - m.put(CF, CQ, new Value(value.getBytes(UTF_8))); + try (BatchWriter bw = accumuloClient.createBatchWriter(opts.getTableName())) { + boolean delete = opts.mode.equals("delete"); + + for (long i = 0; i < opts.num; i++) { + byte[] row = encodeLong(i + opts.start); + String value = "" + (i + opts.start); + + Mutation m = new Mutation(new Text(row)); + if (delete) { + m.putDelete(CF, CQ); + } else { + m.put(CF, CQ, new Value(value.getBytes(UTF_8))); + } + bw.addMutation(m); } - bw.addMutation(m); } - - bw.close(); } else if (opts.mode.equals("verifyDeleted")) { try (Scanner s = accumuloClient.createScanner(opts.getTableName(), opts.auths)) { - s.setBatchSize(scanOpts.scanBatchSize); Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE); Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, @@ -134,7 +128,6 @@ public class TestBinaryRows { Long.MAX_VALUE); Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0); - s.setBatchSize(scanOpts.scanBatchSize); s.setRange(new Range(startKey, stopKey)); long i = opts.start; @@ -169,7 +162,6 @@ public class TestBinaryRows { long row = ((r.nextLong() & 0x7fffffffffffffffL) % opts.num) + opts.start; try (Scanner s = accumuloClient.createScanner(opts.getTableName(), opts.auths)) { - s.setBatchSize(scanOpts.scanBatchSize); Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE); Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0); s.setRange(new Range(startKey, stopKey)); @@ -232,12 +224,10 @@ public class TestBinaryRows { public static void main(String[] args) { Opts opts = new Opts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); - ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts); + opts.parseArgs(TestBinaryRows.class.getName(), args); try (AccumuloClient client = opts.createClient()) { - runTest(client, opts, bwOpts, scanOpts); + runTest(client, opts); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/test/src/main/java/org/apache/accumulo/test/TestIngest.java b/test/src/main/java/org/apache/accumulo/test/TestIngest.java index c2b3900..767e397 100644 --- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java @@ -25,7 +25,6 @@ import java.util.Random; import java.util.Set; import java.util.TreeSet; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.cli.ClientOnDefaultTable; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; @@ -188,16 +187,15 @@ public class TestIngest { public static void main(String[] args) { Opts opts = new Opts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); String name = TestIngest.class.getSimpleName(); TraceUtil.enableClientTraces(null, name, new Properties()); - try (TraceScope clientSpan = opts.parseArgsAndTrace(name, args, bwOpts)) { + try (TraceScope clientSpan = opts.parseArgsAndTrace(name, args)) { if (opts.debug) Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE); try (AccumuloClient client = opts.createClient()) { - ingest(client, opts, bwOpts); + ingest(client, opts); } } catch (Exception e) { throw new RuntimeException(e); @@ -208,9 +206,9 @@ public class TestIngest { @SuppressFBWarnings(value = "PREDICTABLE_RANDOM", justification = "predictable random is okay for testing") - public static void ingest(AccumuloClient accumuloClient, FileSystem fs, Opts opts, - BatchWriterOpts bwOpts) throws IOException, AccumuloException, AccumuloSecurityException, - TableNotFoundException, MutationsRejectedException, TableExistsException { + public static void ingest(AccumuloClient accumuloClient, FileSystem fs, Opts opts) + throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, + MutationsRejectedException, TableExistsException { long stopTime; byte[][] bytevals = generateValues(opts.dataSize); @@ -233,7 +231,7 @@ public class TestIngest { .withTableConfiguration(DefaultConfiguration.getInstance()).build(); writer.startDefaultLocalityGroup(); } else { - bw = accumuloClient.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig()); + bw = accumuloClient.createBatchWriter(opts.getTableName()); accumuloClient.securityOperations().changeUserAuthorizations(opts.getPrincipal(), AUTHS); } Text labBA = new Text(opts.columnVisibility.getExpression()); @@ -353,10 +351,10 @@ public class TestIngest { elapsed); } - public static void ingest(AccumuloClient c, Opts opts, BatchWriterOpts batchWriterOpts) + public static void ingest(AccumuloClient c, Opts opts) throws MutationsRejectedException, IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { ClientContext cc = (ClientContext) c; - ingest(c, FileSystem.get(cc.getHadoopConf()), opts, batchWriterOpts); + ingest(c, FileSystem.get(cc.getHadoopConf()), opts); } } diff --git a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java index 7dc36c3..b9f982d 100644 --- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java @@ -22,9 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map.Entry; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.cli.ClientOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.MutationsRejectedException; @@ -50,15 +48,14 @@ public class TestMultiTableIngest { String prefix = "test_"; } - private static void readBack(Opts opts, ScannerOpts scanOpts, AccumuloClient client, - List tableNames) throws Exception { + private static void readBack(Opts opts, AccumuloClient client, List tableNames) + throws Exception { int i = 0; for (String table : tableNames) { // wait for table to exist while (!client.tableOperations().exists(table)) UtilWaitThread.sleep(100); try (Scanner scanner = client.createScanner(table, opts.auths)) { - scanner.setBatchSize(scanOpts.scanBatchSize); int count = i; for (Entry elt : scanner) { String expected = String.format("%06d", count); @@ -76,9 +73,7 @@ public class TestMultiTableIngest { ArrayList tableNames = new ArrayList<>(); Opts opts = new Opts(); - ScannerOpts scanOpts = new ScannerOpts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); - opts.parseArgs(TestMultiTableIngest.class.getName(), args, scanOpts, bwOpts); + opts.parseArgs(TestMultiTableIngest.class.getName(), args); // create the test table within accumulo try (AccumuloClient accumuloClient = opts.createClient()) { for (int i = 0; i < opts.tables; i++) { @@ -91,7 +86,7 @@ public class TestMultiTableIngest { MultiTableBatchWriter b; try { - b = accumuloClient.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig()); + b = accumuloClient.createMultiTableBatchWriter(); } catch (Exception e) { throw new RuntimeException(e); } @@ -110,7 +105,7 @@ public class TestMultiTableIngest { } } try { - readBack(opts, scanOpts, accumuloClient, tableNames); + readBack(opts, accumuloClient, tableNames); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/test/src/main/java/org/apache/accumulo/test/TestRandomDeletes.java b/test/src/main/java/org/apache/accumulo/test/TestRandomDeletes.java index e943ca9..ad9e75d 100644 --- a/test/src/main/java/org/apache/accumulo/test/TestRandomDeletes.java +++ b/test/src/main/java/org/apache/accumulo/test/TestRandomDeletes.java @@ -22,9 +22,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.cli.ClientOnDefaultTable; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Scanner; @@ -79,12 +77,11 @@ public class TestRandomDeletes { } } - private static TreeSet scanAll(ClientOnDefaultTable opts, ScannerOpts scanOpts, - String tableName) throws Exception { + private static TreeSet scanAll(ClientOnDefaultTable opts, String tableName) + throws Exception { TreeSet result = new TreeSet<>(); try (AccumuloClient client = opts.createClient(); Scanner scanner = client.createScanner(tableName, auths)) { - scanner.setBatchSize(scanOpts.scanBatchSize); for (Entry entry : scanner) { Key key = entry.getKey(); Column column = new Column(TextUtil.getBytes(key.getColumnFamily()), @@ -96,29 +93,27 @@ public class TestRandomDeletes { return result; } - private static long scrambleDeleteHalfAndCheck(ClientOnDefaultTable opts, ScannerOpts scanOpts, - BatchWriterOpts bwOpts, String tableName, Set rows) throws Exception { + private static long scrambleDeleteHalfAndCheck(ClientOnDefaultTable opts, String tableName, + Set rows) throws Exception { int result = 0; ArrayList entries = new ArrayList<>(rows); java.util.Collections.shuffle(entries); - try (AccumuloClient accumuloClient = opts.createClient()) { - BatchWriter mutations = accumuloClient.createBatchWriter(tableName, - bwOpts.getBatchWriterConfig()); + try (AccumuloClient accumuloClient = opts.createClient(); + BatchWriter bw = accumuloClient.createBatchWriter(tableName)) { for (int i = 0; i < (entries.size() + 1) / 2; i++) { RowColumn rc = entries.get(i); Mutation m = new Mutation(rc.row); m.putDelete(new Text(rc.column.columnFamily), new Text(rc.column.columnQualifier), new ColumnVisibility(rc.column.getColumnVisibility()), rc.timestamp + 1); - mutations.addMutation(m); + bw.addMutation(m); rows.remove(rc); result++; } - mutations.close(); } - Set current = scanAll(opts, scanOpts, tableName); + Set current = scanAll(opts, tableName); current.removeAll(rows); if (current.size() > 0) { throw new RuntimeException(current.size() + " records not deleted"); @@ -129,9 +124,7 @@ public class TestRandomDeletes { public static void main(String[] args) { ClientOnDefaultTable opts = new ClientOnDefaultTable("test_ingest"); - ScannerOpts scanOpts = new ScannerOpts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); - opts.parseArgs(TestRandomDeletes.class.getName(), args, scanOpts, bwOpts); + opts.parseArgs(TestRandomDeletes.class.getName(), args); log.info("starting random delete test"); @@ -140,12 +133,12 @@ public class TestRandomDeletes { String tableName = opts.getTableName(); - TreeSet doomed = scanAll(opts, scanOpts, tableName); + TreeSet doomed = scanAll(opts, tableName); log.info("Got {} rows", doomed.size()); long startTime = System.currentTimeMillis(); while (true) { - long half = scrambleDeleteHalfAndCheck(opts, scanOpts, bwOpts, tableName, doomed); + long half = scrambleDeleteHalfAndCheck(opts, tableName, doomed); deleted += half; if (half == 0) break; diff --git a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java index f099fa8..777be8b 100644 --- a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java @@ -22,7 +22,6 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Random; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -65,8 +64,7 @@ public class VerifyIngest { public static void main(String[] args) throws Exception { Opts opts = new Opts(); - ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(VerifyIngest.class.getName(), args, scanOpts); + opts.parseArgs(VerifyIngest.class.getName(), args); if (opts.trace) { TraceUtil.enableClientTraces(null, null, new Properties()); } @@ -77,7 +75,7 @@ public class VerifyIngest { span.addKVAnnotation("cmdLine", Arrays.asList(args).toString()); try (AccumuloClient client = opts.createClient()) { - verifyIngest(client, opts, scanOpts); + verifyIngest(client, opts); } } finally { @@ -87,7 +85,7 @@ public class VerifyIngest { @SuppressFBWarnings(value = "PREDICTABLE_RANDOM", justification = "predictable random is okay for testing") - public static void verifyIngest(AccumuloClient accumuloClient, Opts opts, ScannerOpts scanOpts) + public static void verifyIngest(AccumuloClient accumuloClient, Opts opts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { byte[][] bytevals = TestIngest.generateValues(opts.dataSize); @@ -162,7 +160,6 @@ public class VerifyIngest { Key startKey = new Key(new Text("row_" + String.format("%010d", expectedRow))); try (Scanner scanner = accumuloClient.createScanner(opts.getTableName(), labelAuths)) { - scanner.setBatchSize(scanOpts.scanBatchSize); scanner.setRange(new Range(startKey, endKey)); for (int j = 0; j < opts.cols; j++) { scanner.fetchColumn(new Text(opts.columnFamily), diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java index 36dbe48..b7bdc79 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BalanceInPresenceOfOfflineTableIT.java @@ -25,8 +25,6 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -136,10 +134,10 @@ public class BalanceInPresenceOfOfflineTableIT extends AccumuloClusterHarness { vopts.setClientProperties(getClientProperties()); vopts.rows = opts.rows = 200000; opts.setTableName(TEST_TABLE); - TestIngest.ingest(accumuloClient, opts, new BatchWriterOpts()); + TestIngest.ingest(accumuloClient, opts); accumuloClient.tableOperations().flush(TEST_TABLE, null, null, true); vopts.setTableName(TEST_TABLE); - VerifyIngest.verifyIngest(accumuloClient, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(accumuloClient, vopts); log.debug("waiting for balancing, up to ~5 minutes to allow for migration cleanup."); final long startTime = System.currentTimeMillis(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BinaryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BinaryIT.java index 8c85f3b..ac70384 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BinaryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BinaryIT.java @@ -19,8 +19,6 @@ package org.apache.accumulo.test.functional; import java.util.SortedSet; import java.util.TreeSet; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.test.TestBinaryRows; @@ -57,32 +55,30 @@ public class BinaryIT extends AccumuloClusterHarness { } public static void runTest(AccumuloClient c, String tableName) throws Exception { - BatchWriterOpts bwOpts = new BatchWriterOpts(); - ScannerOpts scanOpts = new ScannerOpts(); TestBinaryRows.Opts opts = new TestBinaryRows.Opts(); opts.setTableName(tableName); opts.start = 0; opts.num = 100000; opts.mode = "ingest"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); opts.mode = "verify"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); opts.start = 25000; opts.num = 50000; opts.mode = "delete"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); opts.start = 0; opts.num = 25000; opts.mode = "verify"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); opts.start = 75000; opts.num = 25000; opts.mode = "randomLookups"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); opts.start = 25000; opts.num = 50000; opts.mode = "verifyDeleted"; - TestBinaryRows.runTest(c, opts, bwOpts, scanOpts); + TestBinaryRows.runTest(c, opts); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java index 4069d2c..b03db9e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java @@ -18,8 +18,6 @@ package org.apache.accumulo.test.functional; import java.io.IOException; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -38,8 +36,6 @@ public class BulkIT extends AccumuloClusterHarness { private static final int N = 100000; private static final int COUNT = 5; - private static final BatchWriterOpts BWOPTS = new BatchWriterOpts(); - private static final ScannerOpts SOPTS = new ScannerOpts(); @Override protected int defaultTimeoutSeconds() { @@ -89,13 +85,13 @@ public class BulkIT extends AccumuloClusterHarness { for (int i = 0; i < COUNT; i++) { opts.outputFile = new Path(files, String.format(fileFormat, i)).toString(); opts.startRow = N * i; - TestIngest.ingest(c, fs, opts, BWOPTS); + TestIngest.ingest(c, fs, opts); } opts.outputFile = new Path(files, String.format(fileFormat, N)).toString(); opts.startRow = N; opts.rows = 1; // create an rfile with one entry, there was a bug with this: - TestIngest.ingest(c, fs, opts, BWOPTS); + TestIngest.ingest(c, fs, opts); bulkLoad(c, tableName, bulkFailures, files, useOld); VerifyIngest.Opts vopts = new VerifyIngest.Opts(); @@ -105,11 +101,11 @@ public class BulkIT extends AccumuloClusterHarness { for (int i = 0; i < COUNT; i++) { vopts.startRow = i * N; vopts.rows = N; - VerifyIngest.verifyIngest(c, vopts, SOPTS); + VerifyIngest.verifyIngest(c, vopts); } vopts.startRow = N; vopts.rows = 1; - VerifyIngest.verifyIngest(c, vopts, SOPTS); + VerifyIngest.verifyIngest(c, vopts); } @SuppressWarnings("deprecation") diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java index ddd2d6b..63b41fe 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java @@ -20,7 +20,6 @@ import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -119,7 +118,7 @@ public class BulkSplitOptimizationIT extends AccumuloClusterHarness { opts.setTableName(tableName); opts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, opts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, opts); // ensure each tablet does not have all map files, should be ~2.5 files per tablet FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java index 0714737..25eac32 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ChaoticBalancerIT.java @@ -20,8 +20,6 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.conf.Property; @@ -70,9 +68,9 @@ public class ChaoticBalancerIT extends AccumuloClusterHarness { vopts.setTableName(tableName); opts.setClientProperties(getClientProperties()); vopts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); c.tableOperations().flush(tableName, null, null, true); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); } } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java index 980ad11..ceca72c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java @@ -25,7 +25,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.InstanceOperations; @@ -140,7 +139,7 @@ public class CompactionIT extends AccumuloClusterHarness { opts.cols = 1; opts.setTableName(tableName); opts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, opts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, opts); } catch (Exception ex) { log.warn("Got exception verifying data", ex); fail.set(true); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java b/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java index 279991b..0d30547 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java @@ -22,8 +22,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.apache.accumulo.cluster.AccumuloCluster; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -72,8 +70,7 @@ public class DeleteIT extends AccumuloClusterHarness { opts.setClientProperties(getClientProperties()); vopts.setClientProperties(getClientProperties()); - BatchWriterOpts BWOPTS = new BatchWriterOpts(); - TestIngest.ingest(c, opts, BWOPTS); + TestIngest.ingest(c, opts); String[] args = null; @@ -90,8 +87,8 @@ public class DeleteIT extends AccumuloClusterHarness { } assertEquals(0, cluster.getClusterControl().exec(TestRandomDeletes.class, args)); - TestIngest.ingest(c, opts, BWOPTS); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + TestIngest.ingest(c, opts); + VerifyIngest.verifyIngest(c, vopts); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java index a1c688f..72d7504 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.Credentials; @@ -90,7 +89,7 @@ public class DynamicThreadPoolsIT extends AccumuloClusterHarness { opts.createTable = true; opts.setTableName(firstTable); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); c.tableOperations().flush(firstTable, null, null, true); for (int i = 1; i < tables.length; i++) c.tableOperations().clone(firstTable, tables[i], true, null, null); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java index 949f598..37d9b66 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -55,7 +54,7 @@ public class FateStarvationIT extends AccumuloClusterHarness { opts.cols = 1; opts.setTableName(tableName); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); c.tableOperations().flush(tableName, null, null, true); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java index 37f52ba..6a76d48 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java @@ -35,7 +35,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.cluster.AccumuloCluster; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.clientImpl.ClientContext; @@ -135,14 +134,11 @@ public class FunctionalTestUtils { opts.rows = rows / splits; opts.startRow = i; opts.cols = 1; - threadPool.execute(new Runnable() { - @Override - public void run() { - try { - TestIngest.ingest(c, fs, opts, new BatchWriterOpts()); - } catch (Exception e) { - fail.set(true); - } + threadPool.execute(() -> { + try { + TestIngest.ingest(c, fs, opts); + } catch (Exception e) { + fail.set(true); } }); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java index 42354a1..05aa813 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java @@ -33,8 +33,6 @@ import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Scanner; @@ -120,7 +118,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase { vopts.cols = opts.cols = 1; opts.setClientProperties(getClientProperties()); vopts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, cluster.getFileSystem(), opts, new BatchWriterOpts()); + TestIngest.ingest(c, cluster.getFileSystem(), opts); c.tableOperations().compact("test_ingest", null, null, true, true); int before = countFiles(); while (true) { @@ -135,7 +133,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase { getCluster().start(); sleepUninterruptibly(15, TimeUnit.SECONDS); int after = countFiles(); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); assertTrue(after < before); } } @@ -146,7 +144,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase { log.info("Filling metadata table with bogus delete flags"); try (AccumuloClient c = createClient()) { - addEntries(c, new BatchWriterOpts()); + addEntries(c); cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE); ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class); sleepUninterruptibly(20, TimeUnit.SECONDS); @@ -299,11 +297,10 @@ public class GarbageCollectorIT extends ConfigurableMacBase { return Iterators.size(Arrays.asList(cluster.getFileSystem().globStatus(path)).iterator()); } - public static void addEntries(AccumuloClient client, BatchWriterOpts bwOpts) throws Exception { + public static void addEntries(AccumuloClient client) throws Exception { client.securityOperations().grantTablePermission(client.whoami(), MetadataTable.NAME, TablePermission.WRITE); - try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME, - bwOpts.getBatchWriterConfig())) { + try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) { for (int i = 0; i < 100000; ++i) { final Text emptyText = new Text(""); Text row = new Text( diff --git a/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java index b86378e..dca075f 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.util.Daemon; @@ -177,7 +176,7 @@ public class HalfDeadTServerIT extends ConfigurableMacBase { VerifyIngest.Opts vopts = new VerifyIngest.Opts(); vopts.rows = rows; vopts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); } else { sleepUninterruptibly(5, TimeUnit.SECONDS); tserver.waitFor(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java index 413f652..9c2bef8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java @@ -19,8 +19,6 @@ package org.apache.accumulo.test.functional; import java.util.Map; import org.apache.accumulo.cluster.ClusterControl; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.conf.Property; @@ -55,7 +53,7 @@ public class MasterFailoverIT extends AccumuloClusterHarness { TestIngest.Opts opts = new TestIngest.Opts(); opts.setTableName(names[0]); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); ClusterControl control = cluster.getClusterControl(); control.stopAllServers(ServerType.MASTER); @@ -66,7 +64,7 @@ public class MasterFailoverIT extends AccumuloClusterHarness { VerifyIngest.Opts vopts = new VerifyIngest.Opts(); vopts.setTableName(names[1]); vopts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); } } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenIT.java index c4f493d..2b790b1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenIT.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Random; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.admin.InstanceOperations; @@ -112,7 +111,7 @@ public class MaxOpenIT extends AccumuloClusterHarness { opts.random = i; opts.setTableName(tableName); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); c.tableOperations().flush(tableName, null, null, true); FunctionalTestUtils.checkRFiles(c, tableName, NUM_TABLETS, NUM_TABLETS, i + 1, i + 1); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java index f6e5754..85f5dd0 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java @@ -55,8 +55,6 @@ import javax.net.ssl.X509TrustManager; import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; @@ -223,7 +221,7 @@ public class ReadWriteIT extends AccumuloClusterHarness { opts.setTableName(tableName); opts.setClientProperties(info.getProperties()); - TestIngest.ingest(accumuloClient, opts, new BatchWriterOpts()); + TestIngest.ingest(accumuloClient, opts); } public static void verify(AccumuloClient accumuloClient, ClientInfo info, int rows, int cols, @@ -233,7 +231,6 @@ public class ReadWriteIT extends AccumuloClusterHarness { private static void verify(AccumuloClient accumuloClient, ClientInfo info, int rows, int cols, int width, int offset, String colf, String tableName) throws Exception { - ScannerOpts scannerOpts = new ScannerOpts(); VerifyIngest.Opts opts = new VerifyIngest.Opts(); opts.rows = rows; opts.cols = cols; @@ -243,7 +240,7 @@ public class ReadWriteIT extends AccumuloClusterHarness { opts.setTableName(tableName); opts.setClientProperties(info.getProperties()); - VerifyIngest.verifyIngest(accumuloClient, opts, scannerOpts); + VerifyIngest.verifyIngest(accumuloClient, opts); } public static String[] args(String... args) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RenameIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RenameIT.java index 2d02c77..07c48e8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RenameIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RenameIT.java @@ -16,8 +16,6 @@ */ package org.apache.accumulo.test.functional; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -37,25 +35,23 @@ public class RenameIT extends AccumuloClusterHarness { String[] tableNames = getUniqueNames(2); String name1 = tableNames[0]; String name2 = tableNames[1]; - BatchWriterOpts bwOpts = new BatchWriterOpts(); - ScannerOpts scanOpts = new ScannerOpts(); TestIngest.Opts opts = new TestIngest.Opts(); opts.createTable = true; opts.setTableName(name1); opts.setClientProperties(cluster.getClientProperties()); try (AccumuloClient c = createAccumuloClient()) { - TestIngest.ingest(c, opts, bwOpts); + TestIngest.ingest(c, opts); c.tableOperations().rename(name1, name2); - TestIngest.ingest(c, opts, bwOpts); + TestIngest.ingest(c, opts); VerifyIngest.Opts vopts = new VerifyIngest.Opts(); vopts.setClientProperties(cluster.getClientProperties()); vopts.setTableName(name2); - VerifyIngest.verifyIngest(c, vopts, scanOpts); + VerifyIngest.verifyIngest(c, vopts); c.tableOperations().delete(name1); c.tableOperations().rename(name2, name1); vopts.setTableName(name1); - VerifyIngest.verifyIngest(c, vopts, scanOpts); + VerifyIngest.verifyIngest(c, vopts); FunctionalTestUtils.assertNoDanglingFateLocks((ClientContext) c, getCluster()); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java index 5291e3b..1fababc 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java @@ -31,8 +31,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.cluster.ClusterUser; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -75,10 +73,8 @@ public class RestartIT extends AccumuloClusterHarness { hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); } - private static final ScannerOpts SOPTS = new ScannerOpts(); private static final VerifyIngest.Opts VOPTS = new VerifyIngest.Opts(); private static final TestIngest.Opts OPTS = new TestIngest.Opts(); - private static final BatchWriterOpts BWOPTS = new BatchWriterOpts(); static { OPTS.rows = VOPTS.rows = 10 * 1000; } @@ -143,7 +139,7 @@ public class RestartIT extends AccumuloClusterHarness { control.stopAllServers(ServerType.MASTER); control.startAllServers(ServerType.MASTER); assertEquals(0, ret.get().intValue()); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); } } @@ -156,7 +152,7 @@ public class RestartIT extends AccumuloClusterHarness { VOPTS.setTableName(tableName); OPTS.setClientProperties(getClientProperties()); VOPTS.setClientProperties(getClientProperties()); - TestIngest.ingest(c, OPTS, BWOPTS); + TestIngest.ingest(c, OPTS); ClusterControl control = getCluster().getClusterControl(); // TODO implement a kill all too? @@ -194,7 +190,7 @@ public class RestartIT extends AccumuloClusterHarness { } } while (masterLockData != null); cluster.start(); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); } } @@ -251,7 +247,7 @@ public class RestartIT extends AccumuloClusterHarness { cluster.start(); assertEquals(0, ret.get().intValue()); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); } } @@ -264,11 +260,11 @@ public class RestartIT extends AccumuloClusterHarness { VOPTS.setTableName(tableName); OPTS.setClientProperties(getClientProperties()); VOPTS.setClientProperties(getClientProperties()); - TestIngest.ingest(c, OPTS, BWOPTS); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + TestIngest.ingest(c, OPTS); + VerifyIngest.verifyIngest(c, VOPTS); cluster.getClusterControl().stopAllServers(ServerType.TABLET_SERVER); cluster.start(); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); } } @@ -295,7 +291,7 @@ public class RestartIT extends AccumuloClusterHarness { c.tableOperations().create(tableName); OPTS.setTableName(tableName); OPTS.setClientProperties(getClientProperties()); - TestIngest.ingest(c, OPTS, BWOPTS); + TestIngest.ingest(c, OPTS); try { getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER); getCluster().getClusterControl().adminStopAll(); @@ -328,9 +324,9 @@ public class RestartIT extends AccumuloClusterHarness { TestIngest.Opts opts = new TestIngest.Opts(); opts.setTableName(tableName); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, BWOPTS); + TestIngest.ingest(c, opts); c.tableOperations().flush(tableName, null, null, false); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); getCluster().stop(); } finally { if (getClusterType() == ClusterType.STANDALONE) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java index 811125f..08f1141 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java @@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.cluster.ClusterUser; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -94,7 +93,6 @@ public class RestartStressIT extends AccumuloClusterHarness { VOPTS = new VerifyIngest.Opts(); VOPTS.rows = 10 * 1000; } - private static final ScannerOpts SOPTS = new ScannerOpts(); @Test public void test() throws Exception { @@ -136,7 +134,7 @@ public class RestartStressIT extends AccumuloClusterHarness { assertEquals(0, retCode.get().intValue()); VOPTS.setTableName(tableName); VOPTS.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, VOPTS, SOPTS); + VerifyIngest.verifyIngest(c, VOPTS); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java index cfc0d74..95008cb 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.clientImpl.ClientContext; @@ -75,7 +74,7 @@ public class SimpleBalancerFairnessIT extends ConfigurableMacBase { TestIngest.Opts opts = new TestIngest.Opts(); opts.rows = 50000; opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); c.tableOperations().flush("test_ingest", null, null, false); sleepUninterruptibly(45, TimeUnit.SECONDS); Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD)); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java index 49239ef..c6e7118 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java @@ -26,8 +26,6 @@ import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import org.apache.accumulo.cluster.ClusterUser; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.InstanceOperations; @@ -135,11 +133,11 @@ public class SplitIT extends AccumuloClusterHarness { opts.setTableName(table); opts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); vopts.rows = opts.rows; vopts.setTableName(table); vopts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); while (c.tableOperations().listSplits(table).size() < 10) { sleepUninterruptibly(15, TimeUnit.SECONDS); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java index eeb704e..3f196c8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java @@ -23,8 +23,6 @@ import static org.junit.Assert.assertTrue; import java.io.FileNotFoundException; import org.apache.accumulo.cluster.AccumuloCluster; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.TableOperations; @@ -73,10 +71,10 @@ public class TableIT extends AccumuloClusterHarness { opts.setClientProperties(getClientProperties()); vopts.setClientProperties(getClientProperties()); opts.setTableName(tableName); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); to.flush(tableName, null, null, true); vopts.setTableName(tableName); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); TableId id = TableId.of(to.tableIdMap().get(tableName)); try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) { s.setRange(new KeyExtent(id, null, null).toMetadataRange()); @@ -95,8 +93,8 @@ public class TableIT extends AccumuloClusterHarness { } assertNull(to.tableIdMap().get(tableName)); to.create(tableName); - TestIngest.ingest(c, opts, new BatchWriterOpts()); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + TestIngest.ingest(c, opts); + VerifyIngest.verifyIngest(c, vopts); to.delete(tableName); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java index 162cfad..76f23fd 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java @@ -16,8 +16,6 @@ */ package org.apache.accumulo.test.functional; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -67,12 +65,12 @@ public class WriteAheadLogIT extends AccumuloClusterHarness { opts.setClientProperties(getClientProperties()); vopts.setClientProperties(getClientProperties()); - TestIngest.ingest(c, opts, new BatchWriterOpts()); + TestIngest.ingest(c, opts); vopts.setTableName(tableName); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER); getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WriteLotsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WriteLotsIT.java index 299490b..9bfa724 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/WriteLotsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/WriteLotsIT.java @@ -21,9 +21,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import org.apache.accumulo.core.cli.BatchWriterOpts; -import org.apache.accumulo.core.cli.ScannerOpts; +import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.test.TestIngest; import org.apache.accumulo.test.VerifyIngest; @@ -38,7 +38,11 @@ public class WriteLotsIT extends AccumuloClusterHarness { @Test public void writeLots() throws Exception { - try (AccumuloClient c = createAccumuloClient()) { + BatchWriterConfig bwConfig = new BatchWriterConfig(); + bwConfig.setMaxMemory(1024L * 1024); + bwConfig.setMaxWriteThreads(2); + try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()) + .batchWriterConfig(bwConfig).build()) { final String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); final AtomicReference ref = new AtomicReference<>(); @@ -47,22 +51,16 @@ public class WriteLotsIT extends AccumuloClusterHarness { new ArrayBlockingQueue<>(THREADS)); for (int i = 0; i < THREADS; i++) { final int index = i; - Runnable r = new Runnable() { - @Override - public void run() { - try { - TestIngest.Opts opts = new TestIngest.Opts(); - opts.startRow = index * 10000; - opts.rows = 10000; - opts.setTableName(tableName); - opts.setClientProperties(getClientProperties()); - BatchWriterOpts bwOpts = new BatchWriterOpts(); - bwOpts.batchMemory = 1024L * 1024; - bwOpts.batchThreads = 2; - TestIngest.ingest(c, opts, new BatchWriterOpts()); - } catch (Exception ex) { - ref.set(ex); - } + Runnable r = () -> { + try { + TestIngest.Opts opts = new TestIngest.Opts(); + opts.startRow = index * 10000; + opts.rows = 10000; + opts.setTableName(tableName); + opts.setClientProperties(getClientProperties()); + TestIngest.ingest(c, opts); + } catch (Exception ex) { + ref.set(ex); } }; tpe.execute(r); @@ -76,7 +74,7 @@ public class WriteLotsIT extends AccumuloClusterHarness { vopts.rows = 10000 * THREADS; vopts.setTableName(tableName); vopts.setClientProperties(getClientProperties()); - VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); + VerifyIngest.verifyIngest(c, vopts); } } diff --git a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java index 43b09e2..7859120 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java @@ -29,7 +29,6 @@ import java.util.UUID; import java.util.zip.CRC32; import java.util.zip.Checksum; -import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.cli.ClientOnDefaultTable; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; @@ -81,10 +80,9 @@ public class ContinuousIngest { public static void main(String[] args) throws Exception { ContinuousOpts opts = new ContinuousOpts(); - BatchWriterOpts bwOpts = new BatchWriterOpts(); ClientOnDefaultTable clientOpts = new ClientOnDefaultTable("ci"); try (TraceScope clientSpan = clientOpts.parseArgsAndTrace(ContinuousIngest.class.getName(), - args, bwOpts, opts)) { + args, opts)) { initVisibilities(opts); @@ -98,8 +96,7 @@ public class ContinuousIngest { "Consult the README and create the table before starting ingest."); } - BatchWriter bw = client.createBatchWriter(clientOpts.getTableName(), - bwOpts.getBatchWriterConfig()); + BatchWriter bw = client.createBatchWriter(clientOpts.getTableName()); bw = TraceProxy.trace(bw, TraceUtil.countSampler(1024)); Random r = new SecureRandom(); diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java index 160fe00..8a504a1 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java @@ -83,11 +83,9 @@ public class RollWALPerformanceIT extends ConfigurableMacBase { log.info("Starting ingest"); final long start = System.nanoTime(); ClientInfo info = ClientInfo.from(c.properties()); - final String args[] = {"-i", info.getInstanceName(), "-z", info.getZooKeepers(), "-u", "root", - "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table", tableName, "--num", - Long.toString(50 * 1000), // 50K 100 byte entries - }; - + // Load 50K 100 byte entries + final String[] args = {"-i", info.getInstanceName(), "-z", info.getZooKeepers(), "-u", "root", + "-p", ROOT_PASSWORD, "--table", tableName, "--num", Long.toString(50 * 1000)}; ContinuousIngest.main(args); final long result = System.nanoTime() - start; log.debug(String.format("Finished in %,d ns", result)); diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java index 4627e91..7fce284 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java @@ -34,7 +34,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.clientImpl.ClientContext; @@ -103,8 +102,7 @@ public class CollectTabletStats { public static void main(String[] args) throws Exception { final CollectOptions opts = new CollectOptions(); - final ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts); + opts.parseArgs(CollectTabletStats.class.getName(), args); String columnsTmp[] = {}; if (opts.columns != null) @@ -222,8 +220,8 @@ public class CollectTabletStats { Test test = new Test(ke) { @Override public int runTest() throws Exception { - return scanTablet(client, opts.getTableName(), opts.auths, scanOpts.scanBatchSize, - ke.getPrevEndRow(), ke.getEndRow(), columns); + return scanTablet(client, opts.getTableName(), opts.auths, ke.getPrevEndRow(), + ke.getEndRow(), columns); } }; tests.add(test); @@ -234,8 +232,7 @@ public class CollectTabletStats { for (final KeyExtent ke : tabletsToTest) { threadPool.submit(() -> { try { - calcTabletStats(client, opts.getTableName(), opts.auths, scanOpts.scanBatchSize, ke, - columns); + calcTabletStats(client, opts.getTableName(), opts.auths, ke, columns); } catch (Exception e) { log.error("Failed to calculate tablet stats.", e); } @@ -522,10 +519,9 @@ public class CollectTabletStats { } private static int scanTablet(AccumuloClient client, String table, Authorizations auths, - int batchSize, Text prevEndRow, Text endRow, String[] columns) throws Exception { + Text prevEndRow, Text endRow, String[] columns) throws Exception { try (Scanner scanner = client.createScanner(table, auths)) { - scanner.setBatchSize(batchSize); scanner.setRange(new Range(prevEndRow, false, endRow, true)); for (String c : columns) { @@ -543,12 +539,11 @@ public class CollectTabletStats { } private static void calcTabletStats(AccumuloClient client, String table, Authorizations auths, - int batchSize, KeyExtent ke, String[] columns) throws Exception { + KeyExtent ke, String[] columns) throws Exception { // long t1 = System.currentTimeMillis(); try (Scanner scanner = client.createScanner(table, auths)) { - scanner.setBatchSize(batchSize); scanner.setRange(new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true)); for (String c : columns) { diff --git a/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java b/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java index a5c71ac..6413f5e 100644 --- a/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java +++ b/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java @@ -18,7 +18,6 @@ package org.apache.accumulo.test.performance.scan; import static org.junit.Assert.assertEquals; -import org.apache.accumulo.core.cli.ScannerOpts; import org.junit.Test; /** @@ -34,8 +33,7 @@ public class CollectTabletStatsTest { String[] args = {"-t", tablename, "--iterations", "2"}; final CollectTabletStats.CollectOptions opts = new CollectTabletStats.CollectOptions(); - final ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts); + opts.parseArgs(CollectTabletStats.class.getName(), args); assertEquals("Check iterations is set, default is 3", 2, opts.iterations); assertEquals("Check tablename is set", 0, tablename.compareTo(opts.getTableName())); @@ -51,8 +49,7 @@ public class CollectTabletStatsTest { String[] args = {"-t", tablename, "--iterations", "2", "--numThreads", "99"}; final CollectTabletStats.CollectOptions opts = new CollectTabletStats.CollectOptions(); - final ScannerOpts scanOpts = new ScannerOpts(); - opts.parseArgs(CollectTabletStats.class.getName(), args, scanOpts); + opts.parseArgs(CollectTabletStats.class.getName(), args); assertEquals("Check iterations is set, default is 3", 2, opts.iterations); assertEquals("Check tablename is set", 0, tablename.compareTo(opts.getTableName()));