hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r1562675 - in /hbase/trunk: hbase-it/src/test/java/org/apache/hadoop/hbase/ hbase-server/src/test/java/org/apache/hadoop/hbase/util/
Date Thu, 30 Jan 2014 03:10:07 GMT
Author: apurtell
Date: Thu Jan 30 03:10:06 2014
New Revision: 1562675

URL: http://svn.apache.org/r1562675
Log:
HBASE-10440 Integration tests fail due to nonce collisions (Sergey Shelukhin)

Modified:
    hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java

Modified: hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java?rev=1562675&r1=1562674&r2=1562675&view=diff
==============================================================================
--- hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java
(original)
+++ hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java
Thu Jan 30 03:10:06 2014
@@ -129,7 +129,7 @@ public class IntegrationTestIngest exten
         Assert.fail(errorMsg);
       }
 
-      ret = loadTool.run(getArgsForLoadTestTool("-update", String.format("60:%d", writeThreads),
+      ret = loadTool.run(getArgsForLoadTestTool("-update", String.format("60:%d:1", writeThreads),
           startKey, numKeys));
       if (0 != ret) {
         String errorMsg = "Update failed with error code " + ret;

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java?rev=1562675&r1=1562674&r2=1562675&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
Thu Jan 30 03:10:06 2014
@@ -92,7 +92,8 @@ public class LoadTestTool extends Abstra
 
   /** Usage string for the update option */
   protected static final String OPT_USAGE_UPDATE =
-      "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
+      "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
+      + ">][:<#whether to ignore nonce collisions=0>]";
 
   protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
       Arrays.toString(BloomType.values());
@@ -168,6 +169,7 @@ public class LoadTestTool extends Abstra
   // Updater options
   protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
   protected int updatePercent;
+  protected boolean ignoreConflicts = false;
   protected boolean isBatchUpdate;
 
   // Reader options
@@ -353,18 +355,22 @@ public class LoadTestTool extends Abstra
     }
 
     if (isUpdate) {
-      String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 2);
+      String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
       int colIndex = 0;
       updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
       if (colIndex < mutateOpts.length) {
         numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
       }
+      if (colIndex < mutateOpts.length) {
+        ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
+      }
 
       isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
 
       System.out.println("Batch updates: " + isBatchUpdate);
       System.out.println("Percent of keys to update: " + updatePercent);
       System.out.println("Updater threads: " + numUpdaterThreads);
+      System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
     }
 
     if (isRead) {
@@ -499,6 +505,7 @@ public class LoadTestTool extends Abstra
         updaterThreads = new MultiThreadedUpdater(dataGen, conf, tableName, updatePercent);
       }
       updaterThreads.setBatchUpdate(isBatchUpdate);
+      updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
     }
 
     if (isRead) {

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java?rev=1562675&r1=1562674&r2=1562675&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdater.java
Thu Jan 30 03:10:06 2014
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.Mu
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.hadoop.hbase.exceptions.OperationConflictException;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
 import org.apache.hadoop.util.StringUtils;
@@ -60,6 +61,7 @@ public class MultiThreadedUpdater extend
 
   private MultiThreadedWriterBase writer = null;
   private boolean isBatchUpdate = false;
+  private boolean ignoreNonceConflicts = false;
   private final double updatePercent;
 
   public MultiThreadedUpdater(LoadTestDataGenerator dataGen, Configuration conf,
@@ -294,16 +296,17 @@ public class MultiThreadedUpdater extend
         }
         totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
       } catch (IOException e) {
+        if (ignoreNonceConflicts && (e instanceof OperationConflictException)) {
+          LOG.info("Detected nonce conflict, ignoring: " + e.getMessage());
+	  totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
+          return;
+        }
         failedKeySet.add(keyBase);
         String exceptionInfo;
         if (e instanceof RetriesExhaustedWithDetailsException) {
           RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException)
e;
           exceptionInfo = aggEx.getExhaustiveDescription();
         } else {
-          StringWriter stackWriter = new StringWriter();
-          PrintWriter pw = new PrintWriter(stackWriter);
-          e.printStackTrace(pw);
-          pw.flush();
           exceptionInfo = StringUtils.stringifyException(e);
         }
         LOG.error("Failed to mutate: " + keyBase + " after " +
@@ -363,4 +366,8 @@ public class MultiThreadedUpdater extend
           + exceptionInfo);
     }
   }
+
+  public void setIgnoreNonceConflicts(boolean value) {
+    this.ignoreNonceConflicts = value;
+  }
 }



Mime
View raw message