kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ale...@apache.org
Subject kudu git commit: [java] add BaseKuduTest.doSetup() method
Date Wed, 05 Jul 2017 19:34:08 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 1070e764d -> 7e7452704


[java] add BaseKuduTest.doSetup() method

Added BaseKuduTest.doSetup() to call from custom @BeforeClass methods.
This is to allow for run-time customization of MiniKuduCluster
parameters such as number of masters, number of tablet servers and
default timeout for various operations.  The new method is needed for
a follow-up commit.

Also did minor clean-up on some of the related files affected by the
change above.

Change-Id: Id283bcb6a6d109839d86932cb807a1da8156f6eb
Reviewed-on: http://gerrit.cloudera.org:8080/7336
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <danburkert@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7e745270
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7e745270
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7e745270

Branch: refs/heads/master
Commit: 7e7452704bb5295196f04423c86d0bc317c25311
Parents: 1070e76
Author: Alexey Serbin <aserbin@cloudera.com>
Authored: Thu Jun 29 13:55:15 2017 -0700
Committer: Alexey Serbin <aserbin@cloudera.com>
Committed: Wed Jul 5 19:32:07 2017 +0000

----------------------------------------------------------------------
 .../org/apache/kudu/client/AsyncKuduClient.java |  4 +-
 .../apache/kudu/client/AsyncKuduScanner.java    |  4 +-
 .../org/apache/kudu/client/BaseKuduTest.java    | 78 +++++++++++---------
 .../java/org/apache/kudu/client/ITClient.java   |  3 +-
 .../org/apache/kudu/client/MiniKuduCluster.java |  4 +-
 .../apache/kudu/client/TestAsyncKuduClient.java |  6 +-
 .../kudu/client/TestAsyncKuduSession.java       |  3 +-
 .../org/apache/kudu/client/TestKuduClient.java  |  4 +-
 .../org/apache/kudu/client/TestKuduSession.java |  2 +-
 .../org/apache/kudu/client/TestNegotiator.java  |  1 -
 .../org/apache/kudu/flume/sink/KuduSink.java    |  1 -
 .../apache/kudu/flume/sink/KuduSinkTest.java    |  1 -
 .../apache/kudu/mapreduce/ITInputFormatJob.java |  1 -
 13 files changed, 52 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 3a83c59..019a3f5 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -126,7 +126,7 @@ public class AsyncKuduClient implements AutoCloseable {
    * a lookup of a single partition (e.g. for a write), or re-looking-up a tablet with
    * stale information.
    */
-  static int FETCH_TABLETS_PER_POINT_LOOKUP = 10;
+  static final int FETCH_TABLETS_PER_POINT_LOOKUP = 10;
   /**
    * The number of tablets to fetch from the master when looking up a range of
    * tablets.
@@ -680,7 +680,7 @@ public class AsyncKuduClient implements AutoCloseable {
   void removeSession(AsyncKuduSession session) {
     synchronized (sessions) {
       boolean removed = sessions.remove(session);
-      assert removed == true;
+      assert removed;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
index 8d42de7..5755158 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -555,9 +555,7 @@ public final class AsyncKuduScanner {
     if (closed) {
       return Deferred.fromResult(null);
     }
-    final Deferred<RowResultIterator> d =
-        client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
-    return d;
+    return client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
   }
 
   /** Callback+Errback invoked when the TabletServer closed our scanner.  */

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
index 190ba06..379fadf 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
@@ -45,15 +45,8 @@ public class BaseKuduTest {
 
   protected static final Logger LOG = LoggerFactory.getLogger(BaseKuduTest.class);
 
-  private static final String NUM_MASTERS_PROP = "NUM_MASTERS";
-  private static final int NUM_TABLET_SERVERS = 3;
-  private static final int DEFAULT_NUM_MASTERS = 3;
-
-  // Number of masters that will be started for this test if we're starting
-  // a cluster.
-  private static final int NUM_MASTERS =
-      Integer.getInteger(NUM_MASTERS_PROP, DEFAULT_NUM_MASTERS);
-
+  // Default timeout/sleep interval for various client operations, waiting for various jobs/threads
+  // to complete, etc.
   protected static final int DEFAULT_SLEEP = 50000;
 
   // Currently not specifying a seed since we want a random behavior when running tests that
@@ -79,28 +72,8 @@ public class BaseKuduTest {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    FakeDNS.getInstance().install();
-
     LOG.info("Setting up before class...");
-
-    miniCluster = miniClusterBuilder
-        .numMasters(NUM_MASTERS)
-        .numTservers(NUM_TABLET_SERVERS)
-        .defaultTimeoutMs(DEFAULT_SLEEP)
-        .build();
-
-    masterAddresses = miniCluster.getMasterAddresses();
-    masterHostPorts = miniCluster.getMasterHostPorts();
-
-    LOG.info("Creating new Kudu client...");
-    client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
-                                .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
-                                .build();
-    syncClient = new KuduClient(client);
-    LOG.info("Waiting for tablet servers...");
-    if (!miniCluster.waitForTabletServers(NUM_TABLET_SERVERS)) {
-      fail("Couldn't get " + NUM_TABLET_SERVERS + " tablet servers running, aborting");
-    }
+    doSetup(Integer.getInteger("NUM_MASTERS", 3), 3);
   }
 
   @AfterClass
@@ -120,6 +93,39 @@ public class BaseKuduTest {
     }
   }
 
+  /**
+   * This method is intended to be called from custom @BeforeClass method to setup Kudu mini
cluster
+   * with the specified parameters. The #BaseKuduTest class calls it in its @BeforeClass
method
+   * with the default parameters.
+   *
+   * @param numMasters number of masters in the cluster to start
+   * @param numTabletServers number of tablet servers in the cluster to start
+   * @throws Exception if something goes wrong
+   */
+  protected static void doSetup(int numMasters, int numTabletServers)
+      throws Exception {
+    FakeDNS.getInstance().install();
+
+    miniCluster = miniClusterBuilder
+        .numMasters(numMasters)
+        .numTservers(numTabletServers)
+        .defaultTimeoutMs(DEFAULT_SLEEP)
+        .build();
+    masterAddresses = miniCluster.getMasterAddresses();
+    masterHostPorts = miniCluster.getMasterHostPorts();
+
+    LOG.info("Creating new Kudu client...");
+    client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
+        .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
+        .build();
+    syncClient = new KuduClient(client);
+
+    LOG.info("Waiting for tablet servers...");
+    if (!miniCluster.waitForTabletServers(numTabletServers)) {
+      fail(String.format("Couldn't get %d tablet servers running, aborting", numTabletServers));
+    }
+  }
+
   protected static KuduTable createTable(String tableName, Schema schema,
                                          CreateTableOptions builder) throws KuduException
{
     LOG.info("Creating table: {}", tableName);
@@ -246,7 +252,7 @@ public class BaseKuduTest {
   }
 
   public static Schema getBasicSchema() {
-    ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(5);
+    ArrayList<ColumnSchema> columns = new ArrayList<>(5);
     columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
     columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
     columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_i", Type.INT32).build());
@@ -306,14 +312,14 @@ public class BaseKuduTest {
     return insert;
   }
 
-  static Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>()
{
+  static final Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>()
{
     @Override
     public Object call(Object arg) throws Exception {
       if (arg == null) return null;
       if (arg instanceof Exception) {
         LOG.warn("Got exception", (Exception) arg);
       } else {
-        LOG.warn("Got an error response back " + arg);
+        LOG.warn("Got an error response back {}", arg);
       }
       return new Exception("Can't recover from error, see previous WARN");
     }
@@ -386,8 +392,8 @@ public class BaseKuduTest {
 
       leader = tablet.getLeaderReplica();
       if (leader == null) {
-        LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept "
+
-        deadlineTracker.getElapsedMillis() + "ms");
+        LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept {}
ms",
+            deadlineTracker.getElapsedMillis());
         Thread.sleep(50);
       }
     }
@@ -424,7 +430,7 @@ public class BaseKuduTest {
           .getPort();
     }
     if (leaderPort == -1) {
-      fail("No leader master found after " + DEFAULT_SLEEP + " ms.");
+      fail(String.format("No leader master found after %d ms", DEFAULT_SLEEP));
     }
     return leaderPort;
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
index 0eaefa5..97d5928 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
@@ -448,8 +448,7 @@ public class ITClient extends BaseKuduTest {
     }
   }
 
-  class UncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
-
+  private class UncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
     @Override
     public void uncaughtException(Thread t, Throwable e) {
       // Only report an error if we're still running, else we'll spam the log.

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
index f536950..16f9b79 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
@@ -583,8 +583,8 @@ public class MiniKuduCluster implements AutoCloseable {
     private int numTservers = 3;
     private int defaultTimeoutMs = 50000;
     private boolean enableKerberos = false;
-    private List<String> extraTserverFlags = new ArrayList<>();
-    private List<String> extraMasterFlags = new ArrayList<>();
+    private final List<String> extraTserverFlags = new ArrayList<>();
+    private final List<String> extraMasterFlags = new ArrayList<>();
 
     public MiniKuduClusterBuilder numMasters(int numMasters) {
       this.numMasters = numMasters;

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
index 685a01e..8153328 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
@@ -31,8 +31,6 @@ import com.google.protobuf.ByteString;
 import com.stumbleupon.async.Deferred;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import org.apache.kudu.ColumnSchema;
 import org.apache.kudu.Common;
@@ -42,8 +40,6 @@ import org.apache.kudu.consensus.Metadata;
 import org.apache.kudu.master.Master;
 
 public class TestAsyncKuduClient extends BaseKuduTest {
-  private static final Logger LOG = LoggerFactory.getLogger(TestAsyncKuduClient.class);
-
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     BaseKuduTest.setUpBeforeClass();
@@ -237,7 +233,7 @@ public class TestAsyncKuduClient extends BaseKuduTest {
    */
   @Test(timeout = 100000)
   public void testCreateTableOutOfOrderPrimaryKeys() throws Exception {
-    ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(6);
+    ArrayList<ColumnSchema> columns = new ArrayList<>(6);
     columns.add(new ColumnSchema.ColumnSchemaBuilder("key_1", Type.INT8).key(true).build());
     columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
     columns.add(new ColumnSchema.ColumnSchemaBuilder("key_2", Type.INT16).key(true).build());

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
index d84f972..41c7c27 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
@@ -565,12 +565,11 @@ public class TestAsyncKuduSession extends BaseKuduTest {
     PartialRow upperBound = schema.newPartialRow();
     upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd);
 
-    AsyncKuduScanner scanner = client.newScannerBuilder(table)
+    return client.newScannerBuilder(table)
         .lowerBound(lowerBound)
         .exclusiveUpperBound(upperBound)
         .setProjectedColumnNames(columnNames)
         .build();
-    return scanner;
   }
 
   private TabletServerErrorPB makeTabletServerError() {

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index bd455fa..f158c64 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -773,9 +773,7 @@ public class TestKuduClient extends BaseKuduTest {
    */
   @Test(timeout = 100000)
   public void testScanNonCoveredTable() throws Exception {
-
-    Schema schema = basicSchema;
-    syncClient.createTable(tableName, schema, getBasicTableOptionsWithNonCoveredRange());
+    syncClient.createTable(tableName, basicSchema, getBasicTableOptionsWithNonCoveredRange());
 
     KuduSession session = syncClient.newSession();
     session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
index 273d818..122ed21 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
@@ -32,7 +32,7 @@ import org.junit.rules.TestName;
 
 public class TestKuduSession extends BaseKuduTest {
   @Rule
-  public TestName name = new TestName();
+  public final TestName name = new TestName();
 
   private KuduTable table;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
index 39f554c..ac3e52f 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
@@ -35,7 +35,6 @@ import javax.net.ssl.SSLEngineResult.HandshakeStatus;
 import javax.net.ssl.SSLException;
 import javax.security.auth.Subject;
 
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.protobuf.ByteString;

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
index 38db309..17706f7 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
@@ -30,7 +30,6 @@ import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
 import org.apache.flume.Channel;
 import org.apache.flume.Context;
 import org.apache.flume.Event;

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
index 99aa6d7..937415b 100644
--- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
@@ -27,7 +27,6 @@ import java.util.HashMap;
 import java.util.List;
 
 import com.google.common.base.Charsets;
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import org.apache.flume.Channel;
 import org.apache.flume.Context;

http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
index 62e8f58..6037a58 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.Job;


Mime
View raw message