accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [accumulo-testing] 02/02: Fix warnings and clean up pom.xml
Date Tue, 30 Jul 2019 21:24:54 GMT
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git

commit 237c0c5a952adcef7f1aad29dcb243c602cadf54
Author: Christopher Tubbs <ctubbsii@apache.org>
AuthorDate: Tue Jul 30 17:19:01 2019 -0400

    Fix warnings and clean up pom.xml
    
    POM updates:
    * Update plugins
    * Add m2e profile for ensuring compiler compliance and ignoring plugins
      in Eclipse that Eclipse doesn't understand
    * Require minimum java version of 1.8 and Maven 3.5
    * Remove unneeded plugins (felix bundle and exec-maven-plugin)
    
    Fix warnings in code:
    * Use try-with-resources for TestEnv
    * Use Double.parseDouble instead of new Double
    * Compute iterable sizes from scanners without unused variables
    * Suppress deprecation warnings where needed
    * Update BulkPlusOne RW test to use new bulk import API
    * Add missing case statement for GET_SUMMARIES in TableOp
    * Remove unused variables
---
 pom.xml                                            | 103 +++++++++++++++-----
 .../testing/continuous/ContinuousMoru.java         |  59 ++++++------
 .../testing/continuous/ContinuousVerify.java       | 104 +++++++++++----------
 .../testing/ingest/BulkImportDirectory.java        |   1 +
 .../accumulo/testing/ingest/VerifyIngest.java      |   5 +-
 .../apache/accumulo/testing/mapreduce/RowHash.java |  60 ++++++------
 .../accumulo/testing/mapreduce/TeraSortIngest.java |  62 ++++++------
 .../performance/tests/ConditionalMutationsPT.java  |  27 +++---
 .../testing/performance/tests/ScanExecutorPT.java  |   3 +-
 .../performance/tests/YieldingScanExecutorPT.java  |   2 +-
 .../testing/randomwalk/bulk/BulkPlusOne.java       |  13 +--
 .../testing/randomwalk/concurrent/BulkImport.java  |   4 +-
 .../testing/randomwalk/concurrent/Config.java      |   4 +-
 .../randomwalk/security/SecurityHelper.java        |   1 -
 .../testing/randomwalk/security/TableOp.java       |   5 +-
 .../testing/randomwalk/shard/BulkInsert.java       |   1 +
 .../testing/randomwalk/shard/SortTool.java         |  15 +--
 17 files changed, 264 insertions(+), 205 deletions(-)

diff --git a/pom.xml b/pom.xml
index 79463d0..9f53552 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,15 +126,19 @@
     <pluginManagement>
       <plugins>
         <plugin>
-          <!-- Allows us to get the apache-ds bundle artifacts -->
-          <groupId>org.apache.felix</groupId>
-          <artifactId>maven-bundle-plugin</artifactId>
-          <version>3.0.1</version>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.8.1</version>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>3.2.1</version>
         </plugin>
         <plugin>
           <groupId>com.github.ekryd.sortpom</groupId>
           <artifactId>sortpom-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
           <configuration>
             <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
             <createBackupFile>false</createBackupFile>
@@ -149,7 +153,7 @@
         <plugin>
           <groupId>net.revelc.code.formatter</groupId>
           <artifactId>formatter-maven-plugin</artifactId>
-          <version>2.8.1</version>
+          <version>2.10.0</version>
           <configuration>
             <configFile>${eclipseFormatterStyle}</configFile>
             <compilerCompliance>${maven.compiler.source}</compilerCompliance>
@@ -171,7 +175,7 @@
         <plugin>
           <groupId>net.revelc.code</groupId>
           <artifactId>impsort-maven-plugin</artifactId>
-          <version>1.2.0</version>
+          <version>1.3.0</version>
           <configuration>
             <removeUnused>true</removeUnused>
             <groups>java.,javax.,org.,com.</groups>
@@ -185,6 +189,30 @@
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <!-- must be same id as in the apache parent pom, to override the version
-->
+            <id>enforce-maven-version</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <phase>validate</phase>
+            <configuration>
+              <rules>
+                <requireMavenVersion>
+                  <version>[3.5.0,)</version>
+                </requireMavenVersion>
+                <requireJavaVersion>
+                  <version>[${maven.compiler.target},)</version>
+                </requireJavaVersion>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
           <optimize>true</optimize>
@@ -201,13 +229,6 @@
         </configuration>
       </plugin>
       <plugin>
-        <!-- Allows us to get the apache-ds bundle artifacts -->
-        <groupId>org.apache.felix</groupId>
-        <artifactId>maven-bundle-plugin</artifactId>
-        <extensions>true</extensions>
-        <inherited>true</inherited>
-      </plugin>
-      <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-failsafe-plugin</artifactId>
         <executions>
@@ -221,18 +242,10 @@
         </executions>
       </plugin>
       <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <version>1.5.0</version>
-        <configuration>
-          <cleanupDaemonThreads>false</cleanupDaemonThreads>
-        </configuration>
-      </plugin>
-      <plugin>
         <!-- This was added to ensure project only uses public API -->
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-checkstyle-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>3.1.0</version>
         <configuration>
           <configLocation>contrib/checkstyle.xml</configLocation>
         </configuration>
@@ -240,7 +253,7 @@
           <dependency>
             <groupId>com.puppycrawl.tools</groupId>
             <artifactId>checkstyle</artifactId>
-            <version>8.18</version>
+            <version>8.23</version>
           </dependency>
         </dependencies>
         <executions>
@@ -351,6 +364,48 @@
       </build>
     </profile>
     <profile>
+      <id>m2e</id>
+      <activation>
+        <property>
+          <name>m2e.version</name>
+        </property>
+      </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
+      <build>
+        <pluginManagement>
+          <plugins>
+            <!--This plugin's configuration is used to store Eclipse m2e settings only.
It has no influence on the Maven build itself.-->
+            <plugin>
+              <groupId>org.eclipse.m2e</groupId>
+              <artifactId>lifecycle-mapping</artifactId>
+              <version>1.0.0</version>
+              <configuration>
+                <lifecycleMappingMetadata>
+                  <pluginExecutions>
+                    <pluginExecution>
+                      <pluginExecutionFilter>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-remote-resources-plugin</artifactId>
+                        <versionRange>[0,)</versionRange>
+                        <goals>
+                          <goal>process</goal>
+                        </goals>
+                      </pluginExecutionFilter>
+                      <action>
+                        <ignore />
+                      </action>
+                    </pluginExecution>
+                  </pluginExecutions>
+                </lifecycleMappingMetadata>
+              </configuration>
+            </plugin>
+          </plugins>
+        </pluginManagement>
+      </build>
+    </profile>
+    <profile>
       <id>jdk-release-flag</id>
       <activation>
         <jdk>[9,)</jdk>
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
index c7d102c..99e2fdf 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
@@ -112,44 +112,45 @@ public class ContinuousMoru extends Configured implements Tool {
 
   @Override
   public int run(String[] args) throws Exception {
+    try (ContinuousEnv env = new ContinuousEnv(args)) {
 
-    ContinuousEnv env = new ContinuousEnv(args);
+      Job job = Job.getInstance(getConf(),
+          this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      job.setInputFormatClass(AccumuloInputFormat.class);
 
-    Job job = Job.getInstance(getConf(),
-        this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
-    job.setJarByClass(this.getClass());
-    job.setInputFormatClass(AccumuloInputFormat.class);
+      int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
+      Set<Range> ranges = env.getAccumuloClient().tableOperations()
+          .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
 
-    int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
-    Set<Range> ranges = env.getAccumuloClient().tableOperations()
-        .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
+      AccumuloInputFormat.configure().clientProperties(env.getClientProps())
+          .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
 
-    AccumuloInputFormat.configure().clientProperties(env.getClientProps())
-        .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
+      job.setMapperClass(CMapper.class);
+      job.setNumReduceTasks(0);
+      job.setOutputFormatClass(AccumuloOutputFormat.class);
 
-    job.setMapperClass(CMapper.class);
-    job.setNumReduceTasks(0);
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
+      AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+          .defaultTable(env.getAccumuloTableName()).store(job);
 
-    AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
-        .defaultTable(env.getAccumuloTableName()).store(job);
+      Configuration conf = job.getConfiguration();
+      conf.setLong(MIN, env.getRowMin());
+      conf.setLong(MAX, env.getRowMax());
+      conf.setInt(MAX_CF, env.getMaxColF());
+      conf.setInt(MAX_CQ, env.getMaxColQ());
+      conf.set(CI_ID, UUID.randomUUID().toString());
+      conf.set("mapreduce.job.classloader", "true");
 
-    Configuration conf = job.getConfiguration();
-    conf.setLong(MIN, env.getRowMin());
-    conf.setLong(MAX, env.getRowMax());
-    conf.setInt(MAX_CF, env.getMaxColF());
-    conf.setInt(MAX_CQ, env.getMaxColQ());
-    conf.set(CI_ID, UUID.randomUUID().toString());
-    conf.set("mapreduce.job.classloader", "true");
-
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
+      job.waitForCompletion(true);
+      return job.isSuccessful() ? 0 : 1;
+    }
   }
 
   public static void main(String[] args) throws Exception {
-    ContinuousEnv env = new ContinuousEnv(args);
-    int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args);
-    if (res != 0)
-      System.exit(res);
+    try (ContinuousEnv env = new ContinuousEnv(args)) {
+      int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args);
+      if (res != 0)
+        System.exit(res);
+    }
   }
 }
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
index 14015e3..5db4350 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
@@ -140,69 +140,71 @@ public class ContinuousVerify extends Configured implements Tool {
   @Override
   public int run(String[] args) throws Exception {
 
-    ContinuousEnv env = new ContinuousEnv(args);
-
-    String tableName = env.getAccumuloTableName();
-
-    Job job = Job.getInstance(getConf(),
-        this.getClass().getSimpleName() + "_" + tableName + "_" + System.currentTimeMillis());
-    job.setJarByClass(this.getClass());
-
-    job.setInputFormatClass(AccumuloInputFormat.class);
-
-    boolean scanOffline = Boolean
-        .parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
-    int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
-    int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
-    String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
-
-    Set<Range> ranges;
-    String clone = "";
-    AccumuloClient client = env.getAccumuloClient();
-    String table;
-
-    if (scanOffline) {
-      Random random = new Random();
-      clone = tableName + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffL));
-      client.tableOperations().clone(tableName, clone, true, new HashMap<>(), new HashSet<>());
-      ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
-      client.tableOperations().offline(clone);
-      table = clone;
-    } else {
-      ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
-      table = tableName;
-    }
+    try (ContinuousEnv env = new ContinuousEnv(args)) {
+
+      String tableName = env.getAccumuloTableName();
+
+      Job job = Job.getInstance(getConf(),
+          this.getClass().getSimpleName() + "_" + tableName + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+
+      job.setInputFormatClass(AccumuloInputFormat.class);
+
+      boolean scanOffline = Boolean
+          .parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
+      int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
+      int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
+      String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
+
+      Set<Range> ranges;
+      String clone = "";
+      AccumuloClient client = env.getAccumuloClient();
+      String table;
+
+      if (scanOffline) {
+        Random random = new Random();
+        clone = tableName + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffL));
+        client.tableOperations().clone(tableName, clone, true, new HashMap<>(), new
HashSet<>());
+        ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
+        client.tableOperations().offline(clone);
+        table = clone;
+      } else {
+        ranges = client.tableOperations().splitRangeByTablets(tableName, new Range(), maxMaps);
+        table = tableName;
+      }
 
-    AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table)
-        .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job);
+      AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table)
+          .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job);
 
-    job.setMapperClass(CMapper.class);
-    job.setMapOutputKeyClass(LongWritable.class);
-    job.setMapOutputValueClass(VLongWritable.class);
+      job.setMapperClass(CMapper.class);
+      job.setMapOutputKeyClass(LongWritable.class);
+      job.setMapOutputValueClass(VLongWritable.class);
 
-    job.setReducerClass(CReducer.class);
-    job.setNumReduceTasks(reducers);
+      job.setReducerClass(CReducer.class);
+      job.setNumReduceTasks(reducers);
 
-    job.setOutputFormatClass(TextOutputFormat.class);
+      job.setOutputFormatClass(TextOutputFormat.class);
 
-    job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
-    job.getConfiguration().set("mapreduce.job.classloader", "true");
+      job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
+      job.getConfiguration().set("mapreduce.job.classloader", "true");
 
-    TextOutputFormat.setOutputPath(job, new Path(outputDir));
+      TextOutputFormat.setOutputPath(job, new Path(outputDir));
 
-    job.waitForCompletion(true);
+      job.waitForCompletion(true);
 
-    if (scanOffline) {
-      client.tableOperations().delete(clone);
+      if (scanOffline) {
+        client.tableOperations().delete(clone);
+      }
+      return job.isSuccessful() ? 0 : 1;
     }
-    return job.isSuccessful() ? 0 : 1;
   }
 
   public static void main(String[] args) throws Exception {
-    ContinuousEnv env = new ContinuousEnv(args);
+    try (ContinuousEnv env = new ContinuousEnv(args)) {
 
-    int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args);
-    if (res != 0)
-      System.exit(res);
+      int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args);
+      if (res != 0)
+        System.exit(res);
+    }
   }
 }
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java b/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
index 2b3a2c3..2da2c62 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/BulkImportDirectory.java
@@ -41,6 +41,7 @@ public class BulkImportDirectory {
     String failures = null;
   }
 
+  @SuppressWarnings("deprecation")
   public static void main(String[] args)
       throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException
{
     final FileSystem fs = FileSystem.get(new Configuration());
diff --git a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
index 992a982..b783e8c 100644
--- a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java
@@ -62,10 +62,9 @@ public class VerifyIngest {
     opts.parseArgs(VerifyIngest.class.getName(), args);
     try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build())
{
       if (opts.trace) {
-        String name = VerifyIngest.class.getSimpleName();
         /*
-         * DistributedTrace.enable(); Trace.on(name); Trace.data("cmdLine",
-         * Arrays.asList(args).toString());
+         * String name = VerifyIngest.class.getSimpleName(); DistributedTrace.enable();
+         * Trace.on(name); Trace.data("cmdLine", Arrays.asList(args).toString());
          */
       }
 
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
index 4722799..45e24f0 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
@@ -58,44 +58,46 @@ public class RowHash extends Configured implements Tool {
 
   @Override
   public int run(String[] args) throws Exception {
-    TestEnv env = new TestEnv(args);
-    Job job = Job.getInstance(getConf());
-    job.setJobName(this.getClass().getName());
-    job.setJarByClass(this.getClass());
-    job.setInputFormatClass(AccumuloInputFormat.class);
+    try (TestEnv env = new TestEnv(args)) {
+      Job job = Job.getInstance(getConf());
+      job.setJobName(this.getClass().getName());
+      job.setJarByClass(this.getClass());
+      job.setInputFormatClass(AccumuloInputFormat.class);
 
-    Properties props = env.getTestProperties();
-    String col = props.getProperty(TestProps.ROWHASH_COLUMN);
-    int idx = col.indexOf(":");
-    Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
-    Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
-    Collection<IteratorSetting.Column> cols = Collections.emptyList();
-    if (cf.getLength() > 0)
-      cols = Collections.singleton(new IteratorSetting.Column(cf, cq));
+      Properties props = env.getTestProperties();
+      String col = props.getProperty(TestProps.ROWHASH_COLUMN);
+      int idx = col.indexOf(":");
+      Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+      Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+      Collection<IteratorSetting.Column> cols = Collections.emptyList();
+      if (cf.getLength() > 0)
+        cols = Collections.singleton(new IteratorSetting.Column(cf, cq));
 
-    String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
-    String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
+      String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
+      String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
 
-    AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
-        .fetchColumns(cols).store(job);
-    AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
-        .defaultTable(outputTable).createTables(true).store(job);
+      AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
+          .fetchColumns(cols).store(job);
+      AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
+          .defaultTable(outputTable).createTables(true).store(job);
 
-    job.getConfiguration().set("mapreduce.job.classloader", "true");
-    job.setMapperClass(HashDataMapper.class);
-    job.setMapOutputKeyClass(Text.class);
-    job.setMapOutputValueClass(Mutation.class);
+      job.getConfiguration().set("mapreduce.job.classloader", "true");
+      job.setMapperClass(HashDataMapper.class);
+      job.setMapOutputKeyClass(Text.class);
+      job.setMapOutputValueClass(Mutation.class);
 
-    job.setNumReduceTasks(0);
+      job.setNumReduceTasks(0);
 
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
+      job.setOutputFormatClass(AccumuloOutputFormat.class);
 
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
+      job.waitForCompletion(true);
+      return job.isSuccessful() ? 0 : 1;
+    }
   }
 
   public static void main(String[] args) throws Exception {
-    TestEnv env = new TestEnv(args);
-    ToolRunner.run(env.getHadoopConfiguration(), new RowHash(), args);
+    try (TestEnv env = new TestEnv(args)) {
+      ToolRunner.run(env.getHadoopConfiguration(), new RowHash(), args);
+    }
   }
 }
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
index ee2b5d0..1961b93 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
@@ -352,42 +352,44 @@ public class TeraSortIngest extends Configured implements Tool {
   }
 
   public static void main(String[] args) throws Exception {
-    TestEnv env = new TestEnv(args);
-    ToolRunner.run(env.getHadoopConfiguration(), new TeraSortIngest(), args);
+    try (TestEnv env = new TestEnv(args)) {
+      ToolRunner.run(env.getHadoopConfiguration(), new TeraSortIngest(), args);
+    }
   }
 
   @Override
   public int run(String[] args) throws Exception {
 
-    TestEnv env = new TestEnv(args);
-
-    Job job = Job.getInstance(getConf());
-    job.setJobName("TeraSortIngest");
-    job.setJarByClass(this.getClass());
-    job.setInputFormatClass(RangeInputFormat.class);
-    job.setMapperClass(SortGenMapper.class);
-    job.setMapOutputKeyClass(Text.class);
-    job.setMapOutputValueClass(Mutation.class);
-    job.setNumReduceTasks(0);
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
-
-    Properties testProps = env.getTestProperties();
-    String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
-    Objects.requireNonNull(tableName);
-
-    AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
-        .defaultTable(tableName).store(job);
-
-    Configuration conf = job.getConfiguration();
-    conf.set("mapreduce.job.classloader", "true");
-    for (Object keyObj : testProps.keySet()) {
-      String key = (String) keyObj;
-      if (key.startsWith(TestProps.TERASORT)) {
-        conf.set(key, testProps.getProperty(key));
+    try (TestEnv env = new TestEnv(args)) {
+
+      Job job = Job.getInstance(getConf());
+      job.setJobName("TeraSortIngest");
+      job.setJarByClass(this.getClass());
+      job.setInputFormatClass(RangeInputFormat.class);
+      job.setMapperClass(SortGenMapper.class);
+      job.setMapOutputKeyClass(Text.class);
+      job.setMapOutputValueClass(Mutation.class);
+      job.setNumReduceTasks(0);
+      job.setOutputFormatClass(AccumuloOutputFormat.class);
+
+      Properties testProps = env.getTestProperties();
+      String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
+      Objects.requireNonNull(tableName);
+
+      AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+          .defaultTable(tableName).store(job);
+
+      Configuration conf = job.getConfiguration();
+      conf.set("mapreduce.job.classloader", "true");
+      for (Object keyObj : testProps.keySet()) {
+        String key = (String) keyObj;
+        if (key.startsWith(TestProps.TERASORT)) {
+          conf.set(key, testProps.getProperty(key));
+        }
       }
-    }
 
-    job.waitForCompletion(true);
-    return job.isSuccessful() ? 0 : 1;
+      job.waitForCompletion(true);
+      return job.isSuccessful() ? 0 : 1;
+    }
   }
 }
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
b/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
index 2d4f8b1..bf86a6a 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ConditionalMutationsPT.java
@@ -22,10 +22,8 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Condition;
 import org.apache.accumulo.core.data.ConditionalMutation;
-import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.testing.performance.Environment;
 import org.apache.accumulo.testing.performance.PerformanceTest;
@@ -35,6 +33,7 @@ import org.apache.hadoop.io.Text;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.hash.Hashing;
 
 public class ConditionalMutationsPT implements PerformanceTest {
@@ -85,7 +84,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 1-19",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "ConditionalMutationsTest: average rate (conditions/sec) to run sequence 1-19");
 
     env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -96,7 +95,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 20-39",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "ConditionalMutationsTest: average rate (conditions/sec)  to run sequence 20-39");
   }
 
@@ -157,7 +156,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 1-19",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "RandomizeConditionalMutationsTest: average rate (conditions/sec)  to run sequence
1-19");
 
     env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -168,7 +167,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 20-39",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "RandomizeConditionalMutationsTest: average rate (conditions/sec)  to run sequence
20-39");
   }
 
@@ -241,7 +240,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 1-19",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "RandomizeBatchScanAndWriteTest: average rate (conditions/sec)  to write and scan
sequence 1-19");
 
     env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -252,7 +251,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate: 20-39",
-        new Double(new DecimalFormat("#0.00").format(rateSum / 20)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "RandomizeBatchScanAndWriteTest: average rate (conditions/sec)  to write and scan
sequence 20-39 post flush");
   }
 
@@ -282,10 +281,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
 
     bs.setRanges(ranges);
 
-    int count = 0;
-    for (Map.Entry<Key,Value> entry : bs) {
-      count++;
-    }
+    int count = Iterables.size(bs);
     if (0 != count) {
       throw new RuntimeException("count = " + count);
     }
@@ -324,7 +320,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate1",
-        new Double(new DecimalFormat("#0.00").format(rateSum / numTest)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
         "SetBlockSizeTest: average rate in conditions/sec");
 
     env.getClient().tableOperations().flush(tableName, null, null, true);
@@ -335,7 +331,7 @@ public class ConditionalMutationsPT implements PerformanceTest {
     }
 
     reportBuilder.result("avgRate2",
-        new Double(new DecimalFormat("#0.00").format(rateSum / numTest)),
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
         "SetBlockSizeTest: average rate in conditions/sec post flush");
 
     env.getClient().tableOperations().compact(tableName, null, null, true, true);
@@ -345,7 +341,8 @@ public class ConditionalMutationsPT implements PerformanceTest {
       rateSum += setBlockSizeTime(cw, numRows, numCols);
     }
 
-    reportBuilder.result("avgRate3", new Double(new DecimalFormat("#0.00").format(rateSum
/ 20)),
+    reportBuilder.result("avgRate3",
+        Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
         "SetBlockSizeTest: average rate in conditions/sec post compaction");
     reportBuilder.parameter("numRows", numRows, "SetBlockSizeTest: The number of rows");
     reportBuilder.parameter("numCols", numCols, "SetBlockSizeTest: The number of columns");
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
index 20911cc..ae9b8c0 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java
@@ -18,6 +18,7 @@
 package org.apache.accumulo.testing.performance.tests;
 
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LongSummaryStatistics;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -161,7 +162,7 @@ public class ScanExecutorPT implements PerformanceTest {
     while (!stop.get()) {
       try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
         scanner.setExecutionHints(hints);
-        for (Entry<Key,Value> entry : scanner) {
+        for (Iterator<Entry<Key,Value>> iter = scanner.iterator(); iter.hasNext();
iter.next()) {
           count++;
           if (stop.get()) {
             return count;
diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
b/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
index 59571fb..8fe424f 100644
--- a/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
+++ b/src/main/java/org/apache/accumulo/testing/performance/tests/YieldingScanExecutorPT.java
@@ -182,7 +182,7 @@ public class YieldingScanExecutorPT implements PerformanceTest {
         scanner.addScanIterator(is);
 
         // scanner.setExecutionHints(hints);
-        for (Entry<Key,Value> entry : scanner) {
+        for (Iterator<Entry<Key,Value>> iter = scanner.iterator(); iter.hasNext();
iter.next()) {
           count++;
           if (stop.get()) {
             return count;
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index 60821e2..55d81c1 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -30,7 +30,6 @@ import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
 import org.apache.accumulo.testing.randomwalk.State;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
@@ -58,9 +57,7 @@ public class BulkPlusOne extends BulkImportTest {
     final FileSystem fs = (FileSystem) state.get("fs");
     final Path dir = new Path(fs.getUri() + "/tmp", "bulk_" + UUID.randomUUID().toString());
     log.debug("Bulk loading from {}", dir);
-    final Path fail = new Path(dir.toString() + "_fail");
     final Random rand = state.getRandom();
-    fs.mkdirs(fail);
     final int parts = rand.nextInt(10) + 1;
 
     TreeSet<Integer> startRows = new TreeSet<>();
@@ -96,15 +93,9 @@ public class BulkPlusOne extends BulkImportTest {
       }
       writer.close();
     }
-    env.getAccumuloClient().tableOperations().importDirectory(Setup.getTableName(), dir.toString(),
-        fail.toString(), true);
+    env.getAccumuloClient().tableOperations().importDirectory(dir.toString())
+        .to(Setup.getTableName()).tableTime(true);
     fs.delete(dir, true);
-    FileStatus[] failures = fs.listStatus(fail);
-    if (failures != null && failures.length > 0) {
-      state.set("bulkImportSuccess", "false");
-      throw new Exception(failures.length + " failure files found importing files from "
+ dir);
-    }
-    fs.delete(fail, true);
     log.debug("Finished bulk import, start rows " + printRows + " last row "
         + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
   }
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
index 1d1d0c1..4dd9736 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/BulkImport.java
@@ -126,8 +126,8 @@ public class BulkImport extends Test {
         bw.close();
       }
 
-      client.tableOperations().importDirectory(tableName, bulkDir, bulkDir + "_f",
-          rand.nextBoolean());
+      client.tableOperations().importDirectory(bulkDir).to(tableName).tableTime(rand.nextBoolean())
+          .load();
 
       log.debug("BulkImported to " + tableName);
     } catch (TableNotFoundException e) {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
index b209d33..992fad6 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
@@ -52,6 +52,8 @@ public class Config extends Test {
     return new Setting(property, min, max);
   }
 
+  @SuppressWarnings("deprecation")
+  Property TSERV_READ_AHEAD_MAXCONCURRENT_deprecated = Property.TSERV_READ_AHEAD_MAXCONCURRENT;
   // @formatter:off
 	Setting[] settings = {
 			s(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, 1, 10),
@@ -69,7 +71,7 @@ public class Config extends Test {
 			s(Property.TSERV_DEFAULT_BLOCKSIZE, 100000, 10000000L),
 			s(Property.TSERV_MAX_IDLE, 10000, 500 * 1000),
 			s(Property.TSERV_MAXMEM, 1000000, 3 * 1024 * 1024 * 1024L),
-			s(Property.TSERV_READ_AHEAD_MAXCONCURRENT, 1, 25),
+			s(TSERV_READ_AHEAD_MAXCONCURRENT_deprecated, 1, 25),
 			s(Property.TSERV_MIGRATE_MAXCONCURRENT, 1, 10),
 			s(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, 10000, 1024 * 1024),
 			s(Property.TSERV_RECOVERY_MAX_CONCURRENT, 1, 100),
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
index b632249..0c8c474 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/security/SecurityHelper.java
@@ -45,7 +45,6 @@ public class SecurityHelper {
 
   private static final String authsMap = "authorizationsCountMap";
   private static final String lastKey = "lastMutationKey";
-  private static final String filesystem = "securityFileSystem";
 
   public static String getTableName(State state) {
     return state.getString(tableName);
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java b/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
index 7e217c3..bf58fdb 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/security/TableOp.java
@@ -240,7 +240,7 @@ public class TableOp extends Test {
             rFileWriter.append(k, new Value("Value".getBytes(UTF_8)));
           rFileWriter.close();
           try {
-            tableOps.importDirectory(tableName, dir.toString(), fail.toString(), true);
+            tableOps.importDirectory(dir.toString()).to(tableName).tableTime(true).load();
           } catch (TableNotFoundException tnfe) {
             if (tableExists)
               throw new AccumuloException("Table didn't exist when it should have: " + tableName);
@@ -293,6 +293,9 @@ public class TableOp extends Test {
           props.setProperty("source", "table");
           DropTable.dropTable(state, env, props);
           break;
+
+        case GET_SUMMARIES:
+          throw new UnsupportedOperationException("GET_SUMMARIES not implemented");
       }
     }
   }
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
index e1c377a..1916e57 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/BulkInsert.java
@@ -142,6 +142,7 @@ public class BulkInsert extends Test {
     fs.delete(new Path(rootDir), true);
   }
 
+  @SuppressWarnings("deprecation")
   private void bulkImport(FileSystem fs, State state, RandWalkEnv env, String tableName,
       String rootDir, String prefix) throws Exception {
     while (true) {
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
index 5206417..91ff5eb 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
@@ -18,8 +18,6 @@ package org.apache.accumulo.testing.randomwalk.shard;
 
 import java.util.Collection;
 
-import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
-import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.hadoop.conf.Configured;
@@ -45,6 +43,7 @@ public class SortTool extends Configured implements Tool {
     this.splits = splits;
   }
 
+  @SuppressWarnings("deprecation")
   @Override
   public int run(String[] args) throws Exception {
     Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
@@ -58,16 +57,20 @@ public class SortTool extends Configured implements Tool {
     job.setInputFormatClass(SequenceFileInputFormat.class);
     SequenceFileInputFormat.setInputPaths(job, seqFile);
 
-    job.setPartitionerClass(KeyRangePartitioner.class);
-    KeyRangePartitioner.setSplitFile(job, splitFile);
+    job.setPartitionerClass(
+        org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.class);
+    org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.setSplitFile(job,
+        splitFile);
 
     job.setMapOutputKeyClass(Key.class);
     job.setMapOutputValueClass(Value.class);
 
     job.setNumReduceTasks(splits.size() + 1);
 
-    job.setOutputFormatClass(AccumuloFileOutputFormat.class);
-    AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
+    job.setOutputFormatClass(
+        org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.class);
+    org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.setOutputPath(job,
+        new Path(outputDir));
 
     job.waitForCompletion(true);
     return job.isSuccessful() ? 0 : 1;


Mime
View raw message