kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wdberke...@apache.org
Subject [33/50] kudu git commit: Fix insert-loadgen example to handle insert errors
Date Wed, 28 Mar 2018 22:25:23 GMT
Fix insert-loadgen example to handle insert errors

This commit makes a few changes:

- multi-threading in insert-loadgen is removed. insert-loadgen used
  separate clients per-thread anyway, so there isn't much advantage to
  running with threads over just starting more instances as separate
  processes. The threading made shutting down on errors harder since the
  shutdown had to be communicated to all threads.
- insert-loadgen now fails on row errors by periodically checking the
  accumulated errors instead of ignoring them.
- All java projects have been updated to use the latest 1.1.0 Kudu
  client.


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4d2c46f2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4d2c46f2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4d2c46f2

Branch: refs/heads/master
Commit: 4d2c46f236562f0631ce5c6020431e0f93a5f55a
Parents: 81157cc
Author: Dan Burkert <danburkert@apache.org>
Authored: Tue Dec 6 20:37:10 2016 -0800
Committer: Dan Burkert <danburkert@apache.org>
Committed: Wed Dec 7 09:51:56 2016 -0800

----------------------------------------------------------------------
 java/collectl/pom.xml                           |  2 +-
 java/insert-loadgen/pom.xml                     |  2 +-
 .../kududb/examples/loadgen/InsertLoadgen.java  | 68 ++++++--------------
 java/java-sample/pom.xml                        |  2 +-
 4 files changed, 22 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/4d2c46f2/java/collectl/pom.xml
----------------------------------------------------------------------
diff --git a/java/collectl/pom.xml b/java/collectl/pom.xml
index 0744dec..3727201 100644
--- a/java/collectl/pom.xml
+++ b/java/collectl/pom.xml
@@ -57,7 +57,7 @@
     <dependency>
       <groupId>org.apache.kudu</groupId>
       <artifactId>kudu-client</artifactId>
-      <version>0.10.0</version>
+      <version>1.1.0</version>
     </dependency>
 
     <!-- for logging messages -->

http://git-wip-us.apache.org/repos/asf/kudu/blob/4d2c46f2/java/insert-loadgen/pom.xml
----------------------------------------------------------------------
diff --git a/java/insert-loadgen/pom.xml b/java/insert-loadgen/pom.xml
index e7a4872..09bc0d1 100644
--- a/java/insert-loadgen/pom.xml
+++ b/java/insert-loadgen/pom.xml
@@ -57,7 +57,7 @@
     <dependency>
       <groupId>org.apache.kudu</groupId>
       <artifactId>kudu-client</artifactId>
-      <version>0.10.0</version>
+      <version>1.1.0</version>
     </dependency>
 
     <!-- for logging messages -->

http://git-wip-us.apache.org/repos/asf/kudu/blob/4d2c46f2/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
----------------------------------------------------------------------
diff --git a/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
index 2b32af1..4b76c34 100644
--- a/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
+++ b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
@@ -1,5 +1,10 @@
 package org.kududb.examples.loadgen;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
 import org.apache.kudu.Schema;
 import org.apache.kudu.Type;
 import org.apache.kudu.client.Insert;
@@ -9,14 +14,7 @@ import org.apache.kudu.client.KuduTable;
 import org.apache.kudu.client.PartialRow;
 import org.apache.kudu.client.SessionConfiguration;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-
 public class InsertLoadgen {
-
   private static class RandomDataGenerator {
     private final Random rng;
     private final int index;
@@ -50,7 +48,7 @@ public class InsertLoadgen {
           row.addInt(index, rng.nextInt(Integer.MAX_VALUE));
           return;
         case INT64:
-        case TIMESTAMP:
+        case UNIXTIME_MICROS:
           row.addLong(index, rng.nextLong());
           return;
         case BINARY:
@@ -76,10 +74,16 @@ public class InsertLoadgen {
     }
   }
 
-  public static void runLoad(String masterHost, String tableName) {
-    KuduClient client = new KuduClient.KuduClientBuilder(masterHost).build();
+  public static void main(String[] args) throws Exception {
+    if (args.length != 2) {
+      System.err.println("Usage: InsertLoadgen kudu_master_host kudu_table");
+      System.exit(1);
+    }
+
+    String masterHost = args[0];
+    String tableName = args[1];
 
-    try {
+    try (KuduClient client = new KuduClient.KuduClientBuilder(masterHost).build()) {
       KuduTable table = client.openTable(tableName);
       Schema schema = table.getSchema();
       List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount());
@@ -89,52 +93,18 @@ public class InsertLoadgen {
 
       KuduSession session = client.newSession();
       session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
-      while (true) {
+      for (int insertCount = 0; ; insertCount++) {
         Insert insert = table.newInsert();
         PartialRow row = insert.getRow();
         for (int i = 0; i < schema.getColumnCount(); i++) {
           generators.get(i).generateColumnData(row);
         }
         session.apply(insert);
-      }
 
-    } catch (Exception e) {
-      e.printStackTrace();
-    } finally {
-      try {
-        client.shutdown();
-      } catch (Exception e) {
-        e.printStackTrace();
-      }
-    }
-  }
-
-  public static void main(String[] args) {
-    if (args.length != 3) {
-      System.err.println("Usage: InsertLoadgen kudu_master_host kudu_table num_threads");
-      System.exit(1);
-    }
-
-    final String masterHost = args[0];
-    final String tableName = args[1];
-    int numThreads = Integer.parseInt(args[2]);
-
-    final CountDownLatch latch = new CountDownLatch(numThreads);
-
-    List<Thread> threads = new ArrayList<>(numThreads);
-    for (int i = 0; i < numThreads; i++) {
-      threads.add(new Thread(new Runnable() {
-        public void run() {
-          runLoad(masterHost, tableName);
-          latch.countDown();
+        if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) {
+          throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString());
         }
-      }));
-      threads.get(i).start();
-    }
-    try {
-      latch.await();
-    } catch (InterruptedException e) {
-      e.printStackTrace();
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/4d2c46f2/java/java-sample/pom.xml
----------------------------------------------------------------------
diff --git a/java/java-sample/pom.xml b/java/java-sample/pom.xml
index a948bf3..b078b04 100644
--- a/java/java-sample/pom.xml
+++ b/java/java-sample/pom.xml
@@ -57,7 +57,7 @@
     <dependency>
       <groupId>org.apache.kudu</groupId>
       <artifactId>kudu-client</artifactId>
-      <version>0.10.0</version>
+      <version>1.1.0</version>
     </dependency>
 
     <!-- for logging messages -->


Mime
View raw message