hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1181929 - in /hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils: DataGenerator.java MultiThreadedReader.java
Date Tue, 11 Oct 2011 17:42:39 GMT
Author: nspiegelberg
Date: Tue Oct 11 17:42:39 2011
New Revision: 1181929

URL: http://svn.apache.org/viewvc?rev=1181929&view=rev
Log:
Modify HBase Load Tester to verify entire data.

Summary:
The HBase Load Tester in the read mode would only verify the
prefix of the data instead of the entire data for a particular row. This
has been modified so as to make the value of any cell in the row to be a
deterministic function of the row key itself. Hence the data put into
the row is randomized with a seed being the row key, but since the seed
is the row key we can always deterministically know what its value
should be when we the row.

Test Plan: 1) Run it for HBase running on a 5 node dev cluster.
Reviewed By: kannan
Reviewers: kannan, nspiegelberg
CC: hbase@lists, pritam, kannan, nspiegelberg
Revert Plan:
Tags:

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

Differential Revision: 300961

Modified:
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/DataGenerator.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/MultiThreadedReader.java

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/DataGenerator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/DataGenerator.java?rev=1181929&r1=1181928&r2=1181929&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/DataGenerator.java
(original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/DataGenerator.java
Tue Oct 11 17:42:39 2011
@@ -26,17 +26,6 @@ import org.apache.hadoop.hbase.util.MD5H
 
 public class DataGenerator {
   static Random random_ = new Random();
-  /* one byte fill pattern */
-  public static final String fill1B_     = "-";
-  /* 64 byte fill pattern */
-  public static final String fill64B_    = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
 ";
-  /* alternate 64 byte fill pattern */
-  public static final String fill64BAlt_ = "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789+-";
-  /* 1K fill pattern */
-  public static final String fill1K_     = fill64BAlt_+fill64BAlt_+fill64BAlt_+fill64BAlt_+
-                                           fill64BAlt_+fill64BAlt_+fill64BAlt_+fill64BAlt_+
-                                           fill64BAlt_+fill64BAlt_+fill64BAlt_+fill64BAlt_+
-                                           fill64BAlt_+fill64BAlt_+fill64BAlt_+fill64BAlt_;
 
   int minDataSize_ = 0;
   int maxDataSize_ = 0;
@@ -54,38 +43,23 @@ public class DataGenerator {
     maxDataSize_ = maxDataSize;
   }
 
+  private static byte[] getDataForKey(String rowKey, int dataSize) {
+    // Need a different local random object since multiple threads might invoke
+    // this method at the same time.
+    Random random = new Random(rowKey.hashCode());
+    byte[] rbytes = new byte[dataSize];
+    random.nextBytes(rbytes);
+    return rbytes;
+  }
+
   public byte[] getDataInSize(long key) {
+    String rowKey = DataGenerator.md5PrefixedKey(key);
     int dataSize = minDataSize_ + random_.nextInt(Math.abs(maxDataSize_ - minDataSize_));
-    StringBuilder sb = new StringBuilder();
-
-    // write the key first
-    int sizeLeft = dataSize;
-    String keyAsString = DataGenerator.md5PrefixedKey(key);
-    sb.append(keyAsString);
-    sizeLeft -= keyAsString.length();
-
-    for(int i = 0; i < sizeLeft/1024; ++i)
-    {
-      sb.append(fill1K_);
-    }
-    sizeLeft = sizeLeft % 1024;
-    for(int i = 0; i < sizeLeft/64; ++i)
-    {
-      sb.append(fill64B_);
-    }
-    sizeLeft = sizeLeft % 64;
-    for(int i = 0; i < dataSize%64; ++i)
-    {
-      sb.append(fill1B_);
-    }
-
-    return sb.toString().getBytes();
+    return getDataForKey(rowKey, dataSize);
   }
 
-  public static boolean verify(String rowKey, String actionId, String data) {
-    if(!data.startsWith(rowKey)) {
-      return false;
-    }
-    return true;
+  public static boolean verify(String rowKey, String actionId, byte[] data) {
+    byte[] expectedData = getDataForKey(rowKey, data.length);
+    return (Bytes.equals(expectedData, data));
   }
 }

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/MultiThreadedReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/MultiThreadedReader.java?rev=1181929&r1=1181928&r2=1181929&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/MultiThreadedReader.java
(original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/MultiThreadedReader.java
Tue Oct 11 17:42:39 2011
@@ -176,10 +176,9 @@ public class MultiThreadedReader extends
             List<KeyValue> keyValues = result.list();
             for(KeyValue kv : keyValues) {
               String actionId = new String(kv.getQualifier());
-              String data = new String(kv.getValue());
 
               // if something does not look right report it
-              if(!DataGenerator.verify(rowKey, actionId, data)) {
+              if (!DataGenerator.verify(rowKey, actionId, kv.getValue())) {
                 reader_.numErrors_.addAndGet(1);
                 LOG.error("Error checking data for key = " + rowKey + ", actionId = " + actionId);
               }



Mime
View raw message