incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1233703 - in /incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk: BulkPlusOne.java Setup.java Verify.java
Date Fri, 20 Jan 2012 00:28:46 GMT
Author: kturner
Date: Fri Jan 20 00:28:46 2012
New Revision: 1233703

URL: http://svn.apache.org/viewvc?rev=1233703&view=rev
Log:
ACCUMULO-334 Made Bulk RW test use random start points for files and log those start points.
 Also made it use better tables names and delete the table name when done.

Modified:
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
Fri Jan 20 00:28:46 2012
@@ -16,8 +16,11 @@
  */
 package org.apache.accumulo.server.test.randomwalk.bulk;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
+import java.util.TreeSet;
 import java.util.UUID;
 
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -34,7 +37,7 @@ import org.apache.log4j.Logger;
 
 public class BulkPlusOne extends BulkTest {
   
-  public static final long LOTS = 100000;
+  public static final int LOTS = 100000;
   public static final int COLS = 10;
   public static final int HEX_SIZE = (int) Math.ceil(Math.log(LOTS) / Math.log(16));
   public static final String FMT = "r%0" + HEX_SIZE + "x";
@@ -47,25 +50,36 @@ public class BulkPlusOne extends BulkTes
     FileSystem fs = (FileSystem) state.get("fs");
     fs.mkdirs(fail);
     int parts = rand.nextInt(10) + 1;
-    long ctr = 0;
-    log.debug("preparing bulk file with " + parts + " parts");
+    
+    TreeSet<Integer> startRows = new TreeSet<Integer>();
+    startRows.add(0);
+    while (startRows.size() < parts)
+      startRows.add(rand.nextInt(LOTS));
+    
+    List<String> printRows = new ArrayList<String>(startRows.size());
+    for (Integer row : startRows)
+      printRows.add(String.format(FMT, row));
+    
+    log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT,
LOTS - 1));
     String cols[] = new String[COLS];
     for (int i = 0; i < cols.length; i++) {
       cols[i] = String.format("%03d", i);
     }
     
+    List<Integer> rows = new ArrayList<Integer>(startRows);
+    rows.add(LOTS);
+
     for (int i = 0; i < parts; i++) {
       FileSKVWriter f = FileOperations.getInstance().openWriter(dir + "/" + String.format("part_%d.",
i) + RFile.EXTENSION, fs, fs.getConf(),
           AccumuloConfiguration.getDefaultConfiguration());
       f.startDefaultLocalityGroup();
-      int end = (int) LOTS / parts;
-      if (i == parts - 1)
-        end = (int) (LOTS - ctr);
-      for (int j = 0; j < end; j++) {
+      int start = rows.get(i);
+      int end = rows.get(i + 1);
+      for (int j = start; j < end; j++) {
+        String row = String.format(FMT, j);
         for (String col : cols) {
-          f.append(new Key(String.format(FMT, ctr), "cf", col), value);
+          f.append(new Key(row, "cf", col), value);
         }
-        ctr++;
       }
       f.close();
     }

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
Fri Jan 20 00:28:46 2012
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.server.test.randomwalk.bulk;
 
+import java.net.InetAddress;
 import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
@@ -46,7 +47,9 @@ public class Setup extends Test {
   @Override
   public void visit(State state, Properties props) throws Exception {
     Random rand = new Random();
-    tableName = Integer.toHexString(Math.abs(rand.nextInt()));
+    String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+    String pid = state.getPid();
+    tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis());
     log.info("Starting bulk test on " + tableName);
     
     List<PerColumnIteratorConfig> aggregators = Collections.singletonList(new PerColumnIteratorConfig(new
Text("cf".getBytes()), null,

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java?rev=1233703&r1=1233702&r2=1233703&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
Fri Jan 20 00:28:46 2012
@@ -51,6 +51,7 @@ public class Verify extends Test {
         throw new Exception("Bad key at " + entry);
     }
     log.info("Test successful on table " + Setup.getTableName());
+    state.getConnector().tableOperations().delete(Setup.getTableName());
   }
   
 }



Mime
View raw message