kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [1/5] kudu git commit: ITBLL: avoid creating client many times during Generator step
Date Fri, 03 Mar 2017 20:53:44 GMT
Repository: kudu
Updated Branches:
  refs/heads/branch-1.3.x 6cd1e0f9f -> 6139cdd69


ITBLL: avoid creating client many times during Generator step

Previously ITBLL was creating a new client for each table to be created.
This changes it to create the client at the top of 'run' and reuse the
client.

Change-Id: Ib3d9f5c974e1090b9367784fe8214a21a6b96b77
Reviewed-on: http://gerrit.cloudera.org:8080/6236
Reviewed-by: Jean-Daniel Cryans <jdcryans@apache.org>
Tested-by: Todd Lipcon <todd@apache.org>
(cherry picked from commit 2f94fb665ca1e4fb750215f7522d06c5dfb0631f)
Reviewed-on: http://gerrit.cloudera.org:8080/6241
Reviewed-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/242fbfb7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/242fbfb7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/242fbfb7

Branch: refs/heads/branch-1.3.x
Commit: 242fbfb7fb5aee33071b22c96ae8d8cf3ec52be5
Parents: 6cd1e0f
Author: Todd Lipcon <todd@apache.org>
Authored: Thu Mar 2 14:20:44 2017 -0800
Committer: Todd Lipcon <todd@apache.org>
Committed: Fri Mar 3 18:38:13 2017 +0000

----------------------------------------------------------------------
 .../tools/IntegrationTestBigLinkedList.java     | 83 +++++++++++---------
 1 file changed, 44 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/242fbfb7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
index 8efac49..19b2ba1 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
@@ -27,6 +27,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
@@ -371,6 +372,9 @@ public class IntegrationTestBigLinkedList extends Configured implements
Tool {
 
     private static final Log LOG = LogFactory.getLog(Generator.class);
 
+    private CommandLineParser parser;
+    private KuduClient client;
+
     static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable>
{
 
       static class GeneratorInputSplit extends InputSplit implements Writable {
@@ -660,57 +664,58 @@ public class IntegrationTestBigLinkedList extends Configured implements
Tool {
 
     public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput,
                    Integer width, Integer wrapMultiplier) throws Exception {
-      int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier);
-      if (ret > 0) {
-        return ret;
+      parser = new CommandLineParser(getConf());
+      client = parser.getClient();
+      try {
+        int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier);
+        if (ret > 0) {
+          return ret;
+        }
+        return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
+      } finally {
+        client.close();
+        client = null;
       }
-      return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
     }
 
-    protected void createTables(int numTablets) throws Exception {
 
+    private void createTables(int numTablets) throws Exception {
       createSchema(getTableName(getConf()), getTableSchema(), numTablets);
       createSchema(getHeadsTable(getConf()), getHeadsTableSchema(), numTablets);
     }
 
-    protected void createSchema(String tableName, Schema schema, int numTablets) throws Exception
{
-      CommandLineParser parser = new CommandLineParser(getConf());
-      KuduClient client = parser.getClient();
-      try {
-        if (numTablets < 1) {
-          numTablets = 1;
-        }
+    private void createSchema(String tableName, Schema schema, int numTablets) throws Exception
{
+      Preconditions.checkNotNull(client);
+      if (numTablets < 1) {
+        numTablets = 1;
+      }
 
-        if (client.tableExists(tableName)) {
-          return;
-        }
+      if (client.tableExists(tableName)) {
+        return;
+      }
 
-        CreateTableOptions builder =
-            new CreateTableOptions().setNumReplicas(parser.getNumReplicas())
-                                    .setRangePartitionColumns(ImmutableList.of("key1", "key2"));
-        if (numTablets > 1) {
-          BigInteger min = BigInteger.valueOf(Long.MIN_VALUE);
-          BigInteger max = BigInteger.valueOf(Long.MAX_VALUE);
-          BigInteger step = max.multiply(BigInteger.valueOf(2))
-              .divide(BigInteger.valueOf(numTablets));
-          LOG.info(min.longValue());
-          LOG.info(max.longValue());
-          LOG.info(step.longValue());
-          PartialRow splitRow = schema.newPartialRow();
-          splitRow.addLong("key2", Long.MIN_VALUE);
-          for (int i = 1; i < numTablets; i++) {
-            long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue();
-            LOG.info("key " + key);
-            splitRow.addLong("key1", key);
-            builder.addSplitRow(splitRow);
-          }
+      CreateTableOptions builder =
+          new CreateTableOptions().setNumReplicas(parser.getNumReplicas())
+                                  .setRangePartitionColumns(ImmutableList.of("key1", "key2"));
+      if (numTablets > 1) {
+        BigInteger min = BigInteger.valueOf(Long.MIN_VALUE);
+        BigInteger max = BigInteger.valueOf(Long.MAX_VALUE);
+        BigInteger step = max.multiply(BigInteger.valueOf(2))
+            .divide(BigInteger.valueOf(numTablets));
+        LOG.info(min.longValue());
+        LOG.info(max.longValue());
+        LOG.info(step.longValue());
+        PartialRow splitRow = schema.newPartialRow();
+        splitRow.addLong("key2", Long.MIN_VALUE);
+        for (int i = 1; i < numTablets; i++) {
+          long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue();
+          LOG.info("key " + key);
+          splitRow.addLong("key1", key);
+          builder.addSplitRow(splitRow);
         }
-
-        client.createTable(tableName, schema, builder);
-      } finally {
-        // Done with this client.
-        client.shutdown();
       }
+
+      client.createTable(tableName, schema, builder);
     }
 
     public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,


Mime
View raw message