accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1350779 - in /accumulo/trunk: server/src/main/java/org/apache/accumulo/server/test/continuous/ test/system/continuous/
Date Fri, 15 Jun 2012 20:19:32 GMT
Author: kturner
Date: Fri Jun 15 20:19:31 2012
New Revision: 1350779

URL: http://svn.apache.org/viewvc?rev=1350779&view=rev
Log:
ACCUMULO-637 a few continuous ingest changes: made # entries written configurable, made table
creation automatic, made log dir creation automatic

Modified:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
    accumulo/trunk/test/system/continuous/continuous-env.sh.example
    accumulo/trunk/test/system/continuous/start-batchwalkers.sh
    accumulo/trunk/test/system/continuous/start-ingest.sh
    accumulo/trunk/test/system/continuous/start-scanners.sh
    accumulo/trunk/test/system/continuous/start-stats.sh
    accumulo/trunk/test/system/continuous/start-walkers.sh

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
Fri Jun 15 20:19:31 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.B
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
@@ -67,11 +68,11 @@ public class ContinuousIngest {
     
     args = processOptions(args);
     
-    if (args.length != 13) {
+    if (args.length != 14) {
       throw new IllegalArgumentException(
           "usage : "
               + ContinuousIngest.class.getName()
-              + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <min> <max> <max colf> <max colq> <max
mem> <max latency> <max threads> <enable checksum>");
+              + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <num> <min> <max> <max colf> <max colq>
<max mem> <max latency> <max threads> <enable checksum>");
     }
     
     if (debugLog != null) {
@@ -89,16 +90,17 @@ public class ContinuousIngest {
     
     String table = args[4];
     
-    long min = Long.parseLong(args[5]);
-    long max = Long.parseLong(args[6]);
-    short maxColF = Short.parseShort(args[7]);
-    short maxColQ = Short.parseShort(args[8]);
-    
-    long maxMemory = Long.parseLong(args[9]);
-    long maxLatency = Integer.parseInt(args[10]);
-    int maxWriteThreads = Integer.parseInt(args[11]);
+    long num = Long.parseLong(args[5]);
+    long min = Long.parseLong(args[6]);
+    long max = Long.parseLong(args[7]);
+    short maxColF = Short.parseShort(args[8]);
+    short maxColQ = Short.parseShort(args[9]);
+    
+    long maxMemory = Long.parseLong(args[10]);
+    long maxLatency = Integer.parseInt(args[11]);
+    int maxWriteThreads = Integer.parseInt(args[12]);
     
-    boolean checksum = Boolean.parseBoolean(args[12]);
+    boolean checksum = Boolean.parseBoolean(args[13]);
     
     if (min < 0 || max < 0 || max <= min) {
       throw new IllegalArgumentException("bad min and max");
@@ -109,6 +111,11 @@ public class ContinuousIngest {
     String path = ZooUtil.getRoot(instance) + Constants.ZTRACERS;
     Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path, localhost, "cingest",
1000));
     
+    if (!conn.tableOperations().exists(table))
+      try {
+        conn.tableOperations().create(table);
+      } catch (TableExistsException tee) {}
+
     BatchWriter bw = conn.createBatchWriter(table, maxMemory, maxLatency, maxWriteThreads);
     bw = Trace.wrapAll(bw, new CountSampler(1024));
     
@@ -133,7 +140,7 @@ public class ContinuousIngest {
     
     long lastFlushTime = System.currentTimeMillis();
     
-    while (true) {
+    out: while (true) {
       // generate first set of nodes
       for (int index = 0; index < flushInterval; index++) {
         long rowLong = genLong(min, max, r);
@@ -152,6 +159,8 @@ public class ContinuousIngest {
       }
       
       lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+      if (count >= num)
+        break out;
       
       // generate subsequent sets of nodes that link to previous set of nodes
       for (int depth = 1; depth < maxDepth; depth++) {
@@ -165,6 +174,8 @@ public class ContinuousIngest {
         }
         
         lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+        if (count >= num)
+          break out;
       }
       
       // create one big linked list, this makes all of the first inserts
@@ -175,6 +186,8 @@ public class ContinuousIngest {
         bw.addMutation(m);
       }
       lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+      if (count >= num)
+        break out;
     }
   }
   

Modified: accumulo/trunk/test/system/continuous/continuous-env.sh.example
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/continuous-env.sh.example?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/continuous-env.sh.example (original)
+++ accumulo/trunk/test/system/continuous/continuous-env.sh.example Fri Jun 15 20:19:31 2012
@@ -32,6 +32,9 @@ DEBUG_WALKER=off
 DEBUG_BATCH_WALKER=off
 DEBUG_SCANNER=off
 
+#the number of entries each client should write
+NUM=9223372036854775807
+
 #the minimum random row to generate
 MIN=0
 

Modified: accumulo/trunk/test/system/continuous/start-batchwalkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-batchwalkers.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-batchwalkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-batchwalkers.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_BATCH_WALKER" = "on" ] ; th
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.log";
 fi
 
-pssh -h batch_walkers.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousBatchWalker
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE
$BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out
2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
+pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo
org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $INSTANCE_NAME
$ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS
>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-ingest.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-ingest.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-ingest.sh (original)
+++ accumulo/trunk/test/system/continuous/start-ingest.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_INGEST" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.log";
 fi
 
-pssh -h ingesters.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM
$MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out
2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
+pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM
$MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out
2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-scanners.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-scanners.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-scanners.sh (original)
+++ accumulo/trunk/test/system/continuous/start-scanners.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_SCANNER" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.log";
 fi
 
-pssh -h scanners.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner
$DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME
$SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
+pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner
$DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME
$SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-stats.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-stats.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-stats.sh (original)
+++ accumulo/trunk/test/system/continuous/start-stats.sh Fri Jun 15 20:19:31 2012
@@ -18,6 +18,8 @@
 
 . continuous-env.sh
 
+mkdir -p $CONTINUOUS_LOG_DIR
+
 CONFIG_OUT=$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_config.out
 
 cat $ACCUMULO_HOME/conf/accumulo-env.sh > $CONFIG_OUT

Modified: accumulo/trunk/test/system/continuous/start-walkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-walkers.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-walkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-walkers.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_WALKER" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.log";
 fi
 
-pssh -h walkers.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err
&" < /dev/null
+pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err
&" < /dev/null
 



Mime
View raw message