accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhava...@apache.org
Subject [3/6] git commit: ACCUMULO-2947 Allow configuration of number of cells per mutation in mem stress test
Date Tue, 01 Jul 2014 19:25:13 GMT
ACCUMULO-2947 Allow configuration of number of cells per mutation in mem stress test

A new switch to the memory stress test Write utility, --max-cells-per-mutation,
controls the maximum number of cells to be written in any mutation. If the maximum is
hit before the configured row width, the current mutation is used and the next mutation
picks up where it left off, in the same row.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75c053d8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75c053d8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75c053d8

Branch: refs/heads/master
Commit: 75c053d8f312020ea0e9ffcc66e26c9322ce380d
Parents: d4d455f
Author: Bill Havanki <bhavanki@cloudera.com>
Authored: Thu Jun 26 17:21:05 2014 -0400
Committer: Bill Havanki <bhavanki@cloudera.com>
Committed: Tue Jul 1 15:20:56 2014 -0400

----------------------------------------------------------------------
 .../test/stress/random/RandomMutations.java     | 21 ++++++++++++++++----
 .../accumulo/test/stress/random/Write.java      |  5 ++++-
 .../test/stress/random/WriteOptions.java        |  3 +++
 test/system/stress/stress-env.sh.example        |  4 ++++
 test/system/stress/writer.sh                    |  2 +-
 5 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
index c4504b2..679b983 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
@@ -21,24 +21,37 @@ import org.apache.accumulo.core.data.Mutation;
 public class RandomMutations extends Stream<Mutation> {
   private final RandomByteArrays rows, column_families, column_qualifiers, values;
   private final RandomWithinRange row_widths;
+  private final int max_cells_per_mutation;
+  private byte[] current_row;
+  private int cells_remaining_in_row;
   
   public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families,
-      RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths)
{
+      RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths,
+      int max_cells_per_mutation) {
     this.rows = rows;
     this.column_families = column_families;
     this.column_qualifiers = column_qualifiers;
     this.values = values;
     this.row_widths = row_widths;
+    this.max_cells_per_mutation = (max_cells_per_mutation > 0 ? max_cells_per_mutation
: Integer.MAX_VALUE);
+
+    current_row = null;
+    cells_remaining_in_row = 0;
   }
 
   // TODO should we care about timestamps?
   @Override
   public Mutation next() {
-    Mutation m = new Mutation(rows.next());
-    final int cells = row_widths.next();
-    for(int i = 0; i < cells; ++i) {
+    if (cells_remaining_in_row == 0) {
+      current_row = rows.next();
+      cells_remaining_in_row = row_widths.next();
+    }
+    Mutation m = new Mutation(current_row);
+    final int cells = Math.min(cells_remaining_in_row, max_cells_per_mutation);
+    for(int i = 1; i <= cells; i++) {
       m.put(column_families.next(), column_qualifiers.next(), values.next());
     }
+    cells_remaining_in_row -= cells;
     return m;
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
index 9c29871..bb679ad 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
@@ -83,7 +83,10 @@ public class Write {
             new RandomWithinRange(
                 opts.row_width_seed,
                 opts.rowWidthMin(),
-                opts.rowWidthMax())));
+                opts.rowWidthMax()),
+            // max cells per mutation
+            opts.max_cells_per_mutation)
+        );
     
     while(true) {
       dw.next();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
index c213528..3e6e647 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
@@ -72,6 +72,9 @@ class WriteOptions extends ClientOnDefaultTable {
   @Parameter(names = "--row-width-seed", description = "seed for generating the number of
cells within a row (a row's \"width\")")
   int row_width_seed = 444;
 
+  @Parameter(names = "--max-cells-per-mutation", description = "maximum number of cells per
mutation; non-positive value implies no limit")
+  int max_cells_per_mutation = -1;
+
   @Parameter(names = "--write-delay", description = "milliseconds to wait between writes")
   long write_delay = 0L;
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/system/stress/stress-env.sh.example
----------------------------------------------------------------------
diff --git a/test/system/stress/stress-env.sh.example b/test/system/stress/stress-env.sh.example
index 1360c67..6efb24f 100644
--- a/test/system/stress/stress-env.sh.example
+++ b/test/system/stress/stress-env.sh.example
@@ -52,5 +52,9 @@ CQ_SEED='--cq-seed 3'
 VALUE_SEED='--value-seed 4'
 ROW_WIDTH_SEED='--row-width-seed 5'
 
+# This is the maximum number of cells to include in each mutation written out.
+# A non-positive value implies no limit.
+MAX_CELLS_PER_MUTATION='--max-cells-per-mutation -1'
+
 # This is the delay in milliseconds between writes. Use <= 0 for no delay.
 WRITE_DELAY='--write-delay 0'

http://git-wip-us.apache.org/repos/asf/accumulo/blob/75c053d8/test/system/stress/writer.sh
----------------------------------------------------------------------
diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh
index 7d9b283..ef26fed 100755
--- a/test/system/stress/writer.sh
+++ b/test/system/stress/writer.sh
@@ -37,6 +37,6 @@ host=$(hostname)
 # TBD - --clear-table option
 
 ${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS
$ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE \
-  $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $WRITE_DELAY \
+  $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $MAX_CELLS_PER_MUTATION
$WRITE_DELAY \
     > $LOG_DIR/${ts}_${host}_writer.out \
     2> $LOG_DIR/${ts}_${host}_writer.err


Mime
View raw message