incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [15/47] git commit: Some changes to output format for tests.
Date Mon, 03 Sep 2012 00:32:19 GMT
Some changes to output format for tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4fd33272
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4fd33272
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4fd33272

Branch: refs/heads/master
Commit: 4fd33272b90e748b7586801597b361d47ac59398
Parents: d4a4353
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Aug 28 20:41:53 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Aug 28 20:41:53 2012 -0400

----------------------------------------------------------------------
 .../blur/testsuite/LoadDataContinuously.java       |   27 ++++++++++----
 .../testsuite/RandomSearchTableContinuously.java   |    7 ++--
 2 files changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4fd33272/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java
b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java
index 53097d8..ff5a5a7 100644
--- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java
+++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java
@@ -25,11 +25,13 @@ import com.nearinfinity.blur.thrift.generated.RowMutationType;
 public class LoadDataContinuously {
 
   private static Random random = new Random();
-  private static List<String> words = new ArrayList<String>();
+  public static List<String> words = new ArrayList<String>();
 
   public static void main(String[] args) throws BlurException, TException, IOException {
     if (!(args.length == 8 || args.length == 9)) {
-      System.err.println(LoadDataContinuously.class.getName() + " <host1:port1,host2:port2>
<table name> <WAL true|false> <# of columns per record> <# of records
per row> <# of column families> <# of words per record> <time in seconds
between reporting progress> <*optional path to word dictionary>");
+      System.err
+          .println(LoadDataContinuously.class.getName()
+              + " <host1:port1,host2:port2> <table name> <WAL true|false>
<# of columns per record> <# of records per row> <# of column families>
<# of words per record> <time in seconds between reporting progress> <*optional
path to word dictionary>");
       System.exit(1);
     }
     if (args.length == 9) {
@@ -37,7 +39,7 @@ public class LoadDataContinuously {
     } else {
       loadWords(null);
     }
-    
+
     final Iface client = BlurClient.getClient(args[0]);
     final String table = args[1];
     final boolean wal = Boolean.parseBoolean(args[2]);
@@ -51,12 +53,14 @@ public class LoadDataContinuously {
     long s = start;
     long recordCountTotal = 0;
     long rowCount = 0;
-    
+
     int batchSize = 100;
-    
+
     List<RowMutation> batch = new ArrayList<RowMutation>();
-    
+
     long recordCount = 0;
+    long totalTime = 0;
+    long calls = 0;
     while (true) {
       long now = System.currentTimeMillis();
       if (s + timeBetweenReporting < now) {
@@ -64,9 +68,12 @@ public class LoadDataContinuously {
         double seconds = (now - s) / 1000.0;
         double avgRate = recordCountTotal / avgSeconds;
         double rate = recordCount / seconds;
-        System.out.println("Records indexed [" + recordCountTotal + "] Rows indexed [" +
rowCount + "] at record rate [" + rate + "/s] with record avg rate [" + avgRate + "/s]");
+        double latency = (totalTime / 1000000.0) / calls;
+        System.out.println(System.currentTimeMillis() + "," + recordCountTotal + "," + rowCount
+ "," + latency + "," + rate + "," + avgRate);
         s = now;
         recordCount = 0;
+        totalTime = 0;
+        calls = 0;
       }
 
       RowMutation mutation = new RowMutation();
@@ -80,7 +87,11 @@ public class LoadDataContinuously {
       }
       batch.add(mutation);
       if (batch.size() >= batchSize) {
+        long sm = System.nanoTime();
         client.mutateBatch(batch);
+        long em = System.nanoTime();
+        calls++;
+        totalTime += (em - sm);
         batch.clear();
       }
       rowCount++;
@@ -89,7 +100,7 @@ public class LoadDataContinuously {
     }
   }
 
-  private static void loadWords(String path) throws IOException {
+  public static void loadWords(String path) throws IOException {
     InputStream inputStream;
     if (path == null) {
       inputStream = LoadDataContinuously.class.getResourceAsStream("words.txt");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4fd33272/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java
b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java
index 13c09f2..9486fcc 100644
--- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java
+++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java
@@ -51,7 +51,8 @@ public class RandomSearchTableContinuously {
     int count = 0;
     long resultCount = 0;
     Iface client = BlurClient.getClient(connectionStr);
-    for (int i = 0; i < numberOfSearchesPerPass; i++) {
+    int i;
+    for (i = 0; i < numberOfSearchesPerPass; i++) {
       long now = System.currentTimeMillis();
       if (s + timeBetweenReporting < now) {
         double avgSeconds = (now - start) / 1000.0;
@@ -59,8 +60,7 @@ public class RandomSearchTableContinuously {
         double avgRate = i / avgSeconds;
         double rate = count / seconds;
         double responseTimeAvg = (responseTime / (double) count) / 1000000.0;
-        System.out.println("Searches [" + i + "] at avg response time of [" + responseTimeAvg
+ " ms] at rate [" + rate + "/s] with avg rate [" + avgRate + "/s] results [" + resultCount
+ "]");
-        System.out.println("Record Count [" + getCount(client,tableName) + "]");
+        System.out.println(System.currentTimeMillis() + "," + i + "," + responseTimeAvg +
"," + rate + "," + avgRate + "," + resultCount + "," + getCount(client,tableName));
         s = now;
         responseTime = 0;
         count = 0;
@@ -82,7 +82,6 @@ public class RandomSearchTableContinuously {
       responseTime += (qe - qs);
       count++;
     }
-    
   }
 
   private static long getCount(Iface client, String tableName) throws BlurException, TException
{


Mime
View raw message