accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ujustgotbi...@apache.org
Subject [2/4] git commit: ACCUMULO-2789 Added parameter to control the number of cells within a row
Date Fri, 23 May 2014 23:26:05 GMT
ACCUMULO-2789 Added parameter to control the number of cells within a row

When running the stress test, it is now possible to configure the writer to
send mutations for rows that contain a random amount of column updates. This is
controlled via the --min-row-width, --max-row-width and --row-width-seed
parameters.

I did some internal shuffling to make randomly generating numbers within a
range more convenient. See the new RandomWithinRange class for details.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f0dcf1e5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f0dcf1e5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f0dcf1e5

Branch: refs/heads/1.6.1-SNAPSHOT
Commit: f0dcf1e5570c12098e716ec58ea2f4bc3321053b
Parents: 6970d73
Author: Bill Slacum <ujustgotbilld@apache.org>
Authored: Fri May 23 19:11:39 2014 -0400
Committer: Bill Slacum <ujustgotbilld@apache.org>
Committed: Fri May 23 19:11:39 2014 -0400

----------------------------------------------------------------------
 .../random/RandomByteArrayMakerFunction.java    | 60 --------------------
 .../test/stress/random/RandomByteArrays.java    | 14 ++---
 .../test/stress/random/RandomMutations.java     | 11 +++-
 .../test/stress/random/RandomWithinRange.java   | 43 ++++++++++++++
 .../accumulo/test/stress/random/Write.java      | 33 +++++++----
 .../test/stress/random/WriteOptions.java        | 17 ++++++
 test/system/stress/writer.sh                    |  4 +-
 7 files changed, 97 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
deleted file mode 100644
index bfe0be8..0000000
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.stress.random;
-
-import java.util.Random;
-
-import com.google.common.base.Preconditions;
-
-/**
- * This class is one RBAMF. Don't mess with it, or it'll mess with you.
- */
-public class RandomByteArrayMakerFunction {
-  private final Random random;
-  
-  public RandomByteArrayMakerFunction() {
-    this(0);
-  }
-  
-  public RandomByteArrayMakerFunction(int seed) {
-    random = new Random(seed);
-  }
-  
-  public byte[] make(int size) {
-    byte[] b = new byte[size];
-    random.nextBytes(b);
-    return b;
-  }
-  
-  public byte[] makeWithRandomSize(int max) {
-    return makeWithRandomSize(1, random.nextInt(max));
-  }
-  
-  public byte[] makeWithRandomSize(int min, int max) {
-    Preconditions.checkArgument(min > 0, "Min must be positive.");
-    Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min.");
-    if (min == max) {
-      return make(min);
-    } else {
-      final int spread = max - min;
-      final int random_value = random.nextInt(spread);
-      // we pick a random number that's between 0 and (max - min), then add
-      // min as an offset to get a random number that's [min, max)
-      return make(random_value + min);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
index 62f7cb2..0b6b36a 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
@@ -17,20 +17,16 @@
 package org.apache.accumulo.test.stress.random;
 
 /**
- * An iterator that will create random byte arrays as it is looped over.
+ * A stream that will create random byte arrays as it is looped over.
  */
 public class RandomByteArrays extends Stream<byte[]> {
-  private final RandomByteArrayMakerFunction rbamf;
-  private final int min_array_size, max_array_size;
+  private final RandomWithinRange random_arrays;
   
-  public RandomByteArrays(RandomByteArrayMakerFunction rbamf, int min_array_size,
-      int max_array_size) {
-    this.rbamf = rbamf;
-    this.min_array_size = min_array_size;
-    this.max_array_size = max_array_size;
+  public RandomByteArrays(RandomWithinRange random_arrays) {
+    this.random_arrays = random_arrays;
   }
   
   public byte[] next() {
-    return rbamf.makeWithRandomSize(min_array_size, max_array_size);
+    return random_arrays.next_bytes();
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
index a22222b..c4504b2 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
@@ -20,20 +20,25 @@ import org.apache.accumulo.core.data.Mutation;
 
 public class RandomMutations extends Stream<Mutation> {
   private final RandomByteArrays rows, column_families, column_qualifiers, values;
+  private final RandomWithinRange row_widths;
   
   public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families,
-      RandomByteArrays column_qualifiers, RandomByteArrays values) {
+      RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths)
{
     this.rows = rows;
     this.column_families = column_families;
     this.column_qualifiers = column_qualifiers;
     this.values = values;
+    this.row_widths = row_widths;
   }
-  
+
   // TODO should we care about timestamps?
   @Override
   public Mutation next() {
     Mutation m = new Mutation(rows.next());
-    m.put(column_families.next(), column_qualifiers.next(), values.next());
+    final int cells = row_widths.next();
+    for(int i = 0; i < cells; ++i) {
+      m.put(column_families.next(), column_qualifiers.next(), values.next());
+    }
     return m;
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
new file mode 100644
index 0000000..c66cf38
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
@@ -0,0 +1,43 @@
+package org.apache.accumulo.test.stress.random;
+
+import java.util.Random;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Class that returns positive integers between some minimum
+ * and maximum.
+ *
+ */
+public class RandomWithinRange {
+  private final Random random;
+  private final int min, max;
+  
+  public RandomWithinRange(int seed, int min, int max) {
+    this(new Random(seed), min, max);
+  }
+  
+  public RandomWithinRange(Random random, int min, int max) {
+    Preconditions.checkArgument(min > 0, "Min must be positive.");
+    Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min.");
+    this.random = random;
+    this.min = min;
+    this.max = max;
+  }
+  
+  public int next() {
+    if (min == max) {
+      return min;
+    } else {
+      // we pick a random number that's between 0 and (max - min), then add
+      // min as an offset to get a random number that's [min, max)
+      return random.nextInt(max - min) + min;
+    }
+  }
+  
+  public byte[] next_bytes() {
+    byte[] b = new byte[next()];
+    random.nextBytes(b);
+    return b;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
index 2c817c3..3df9808 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
@@ -52,24 +52,33 @@ public class Write {
         new RandomMutations(
             //rows
             new RandomByteArrays(
-                new RandomByteArrayMakerFunction(opts.row_seed),
-                opts.rowMin(),
-                opts.rowMax()),
+                new RandomWithinRange(
+                    opts.row_seed,
+                    opts.rowMin(),
+                    opts.rowMax())),
             //cfs
             new RandomByteArrays(
-                new RandomByteArrayMakerFunction(opts.cf_seed),
-                opts.cfMin(),
-                opts.cfMax()),
+                new RandomWithinRange(
+                    opts.cf_seed,
+                    opts.cfMin(),
+                    opts.cfMax())),
             //cqs
             new RandomByteArrays(
-                new RandomByteArrayMakerFunction(opts.cq_seed),
-                opts.cqMin(),
-                opts.cqMax()),
+                new RandomWithinRange(
+                    opts.cq_seed,
+                    opts.cqMin(),
+                    opts.cqMax())),
             //vals
             new RandomByteArrays(
-                new RandomByteArrayMakerFunction(opts.value_seed),
-                opts.valueMin(),
-                opts.valueMax())));
+                new RandomWithinRange(
+                    opts.value_seed,
+                    opts.valueMin(),
+                    opts.valueMax())),
+            //number of cells per row
+            new RandomWithinRange(
+                opts.row_width_seed,
+                opts.rowWidthMin(),
+                opts.rowWidthMax())));
     
     while(true) {
       dw.next();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
index b850e85..85ff25b 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
@@ -48,6 +48,12 @@ class WriteOptions extends ClientOnDefaultTable {
   @Parameter(validateValueWith = IntArgValidator.class, names = "--max-value-size", description
= "maximum value size")
   Integer value_max;
   
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-width", description
= "minimum row width")
+  Integer row_width_min;
+  
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-width", description
= "maximum row width")
+  Integer row_width_max;
+  
   @Parameter(names = "--clear-table", description = "clears the table before ingesting")
   boolean clear_table;
   
@@ -63,6 +69,9 @@ class WriteOptions extends ClientOnDefaultTable {
   @Parameter(names = "--value-seed", description = "seed for generating values")
   int value_seed = 99;
   
+  @Parameter(names = "--row-width-seed", description = "seed for generating the number of
cells within a row (a row's \"width\")")
+  int row_width_seed = 444;
+  
   public WriteOptions(String table) {
     super(table);
   }
@@ -149,4 +158,12 @@ class WriteOptions extends ClientOnDefaultTable {
   public int valueMax() {
     return calculateMax(value_min, value_max);
   }
+  
+  public int rowWidthMin() {
+    return minOrDefault(row_width_min);
+  }
+  
+  public int rowWidthMax() {
+    return calculateMax(row_width_min, row_width_max);
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/system/stress/writer.sh
----------------------------------------------------------------------
diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh
index 6054ab1..8265a68 100755
--- a/test/system/stress/writer.sh
+++ b/test/system/stress/writer.sh
@@ -24,15 +24,17 @@ ROW_RANGE='--min-row-size 128 --max-row-size 128'
 CF_RANGE='--min-cf-size 128 --max-cf-size 128'
 CQ_RANGE='--min-cq-size 128 --max-cq-size 128'
 VALUE_RANGE='--min-value-size 1024 --max-value-size 2048'
+ROW_WIDTH='--min-row-width 1 --max-row-width 10'
 
 # These are the seeds for the random number generates used to generate each cell component.
 ROW_SEED='--row-seed 1'
 CF_SEED='--cf-seed 2'
 CQ_SEED='--cq-seed 3'
 VALUE_SEED='--value-seed 4'
+ROW_WIDTH_SEED='--row-width-seed 5'
 
 # Let's reset the table, for good measure
 ../../../bin/accumulo shell $USERPASS -e 'deletetable -f stress_test'
 ../../../bin/accumulo shell $USERPASS -e 'createtable stress_test'
 
-../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE
$CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED
+../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE
$CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED


Mime
View raw message