accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1397746 - in /accumulo/trunk: server/src/main/java/org/apache/accumulo/server/test/continuous/ server/src/main/java/org/apache/accumulo/server/test/scalability/ test/system/continuous/
Date Sat, 13 Oct 2012 00:39:08 GMT
Author: kturner
Date: Sat Oct 13 00:39:07 2012
New Revision: 1397746

URL: http://svn.apache.org/viewvc?rev=1397746&view=rev
Log:
ACCUMULO-810 added authorizations to continuous ingest test

Modified:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java
    accumulo/trunk/test/system/continuous/continuous-env.sh.example
    accumulo/trunk/test/system/continuous/run-verify.sh
    accumulo/trunk/test/system/continuous/start-batchwalkers.sh
    accumulo/trunk/test/system/continuous/start-ingest.sh
    accumulo/trunk/test/system/continuous/start-scanners.sh
    accumulo/trunk/test/system/continuous/start-walkers.sh

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
Sat Oct 13 00:39:07 2012
@@ -32,7 +32,9 @@ import org.apache.accumulo.core.client.Z
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.test.continuous.ContinuousWalk.RandomAuths;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Level;
@@ -41,6 +43,7 @@ import org.apache.log4j.PatternLayout;
 
 public class ContinuousBatchWalker {
   private static String debugLog = null;
+  private static String authsFile = null;
   
   private static String[] processOptions(String[] args) {
     ArrayList<String> al = new ArrayList<String>();
@@ -48,6 +51,8 @@ public class ContinuousBatchWalker {
     for (int i = 0; i < args.length; i++) {
       if (args[i].equals("--debug")) {
         debugLog = args[++i];
+      } else if (args[i].equals("--auths")) {
+        authsFile = args[++i];
       } else {
         al.add(args[i]);
       }
@@ -62,7 +67,7 @@ public class ContinuousBatchWalker {
     
     if (args.length != 10) {
       throw new IllegalArgumentException("usage : " + ContinuousBatchWalker.class.getName()
-          + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <min> <max> <sleep time> <batch size> <query
threads>");
+          + " [--debug <debug log>] [--auths <file>] <instance name> <zookeepers>
<user> <pass> <table> <min> <max> <sleep time> <batch
size> <query threads>");
     }
     
     if (debugLog != null) {
@@ -88,12 +93,14 @@ public class ContinuousBatchWalker {
     int batchSize = Integer.parseInt(args[8]);
     int numQueryThreads = Integer.parseInt(args[9]);
     
-    Connector conn = new ZooKeeperInstance(instanceName, zooKeepers).getConnector(user, password.getBytes());
-    Scanner scanner = conn.createScanner(table, Constants.NO_AUTHS);
-    BatchScanner bs = conn.createBatchScanner(table, Constants.NO_AUTHS, numQueryThreads);
-    
     Random r = new Random();
-    
+    RandomAuths randomAuths = new RandomAuths(authsFile);
+    Authorizations auths = randomAuths.getAuths(r);
+
+    Connector conn = new ZooKeeperInstance(instanceName, zooKeepers).getConnector(user, password.getBytes());
+    Scanner scanner = conn.createScanner(table, auths);
+    BatchScanner bs = conn.createBatchScanner(table, auths, numQueryThreads);
+
     while (true) {
       Set<Text> batch = getBatch(scanner, min, max, batchSize, r);
       

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
Sat Oct 13 00:39:07 2012
@@ -16,8 +16,12 @@
  */
 package org.apache.accumulo.server.test.continuous;
 
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
 import java.net.InetAddress;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
@@ -38,8 +42,12 @@ import org.apache.accumulo.core.client.T
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.server.test.FastFormat;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Level;
@@ -49,15 +57,44 @@ import org.apache.log4j.PatternLayout;
 
 public class ContinuousIngest {
   
+  private static String visFile = null;
   private static String debugLog = null;
   private static final byte[] EMPTY_BYTES = new byte[0];
   
+  private static List<ColumnVisibility> visibilities;
+  
+  private static void initVisibilities() throws Exception {
+    if (visFile == null) {
+      visibilities = Collections.singletonList(new ColumnVisibility());
+      return;
+    }
+    
+    visibilities = new ArrayList<ColumnVisibility>();
+    
+    FileSystem fs = FileSystem.get(new Configuration());
+    BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(visFile))));
+    
+    String line;
+    
+    while ((line = in.readLine()) != null) {
+      visibilities.add(new ColumnVisibility(line));
+    }
+    
+    in.close();
+  }
+
+  private static ColumnVisibility getVisibility(Random rand) {
+    return visibilities.get(rand.nextInt(visibilities.size()));
+  }
+
   private static String[] processOptions(String[] args) {
     ArrayList<String> al = new ArrayList<String>();
     
     for (int i = 0; i < args.length; i++) {
       if (args[i].equals("--debug")) {
         debugLog = args[++i];
+      } else if (args[i].equals("--visibilities")) {
+        visFile = args[++i];
       } else {
         al.add(args[i]);
       }
@@ -74,7 +111,7 @@ public class ContinuousIngest {
       throw new IllegalArgumentException(
           "usage : "
               + ContinuousIngest.class.getName()
-              + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <num> <min> <max> <max colf> <max colq>
<max mem> <max latency> <max threads> <enable checksum>");
+              + " [--debug <debug log>] [--visibilities <file>] <instance
name> <zookeepers> <user> <pass> <table> <num> <min>
<max> <max colf> <max colq> <max mem> <max latency> <max
threads> <enable checksum>");
     }
     
     if (debugLog != null) {
@@ -84,6 +121,8 @@ public class ContinuousIngest {
       logger.addAppender(new FileAppender(new PatternLayout("%d{dd HH:mm:ss,SSS} [%-8c{2}]
%-5p: %m%n"), debugLog, true));
     }
     
+    initVisibilities();
+
     String instanceName = args[0];
     String zooKeepers = args[1];
     
@@ -145,6 +184,8 @@ public class ContinuousIngest {
     
     out: while (true) {
       // generate first set of nodes
+      ColumnVisibility cv = getVisibility(r);
+
       for (int index = 0; index < flushInterval; index++) {
         long rowLong = genLong(min, max, r);
         prevRows[index] = rowLong;
@@ -156,7 +197,7 @@ public class ContinuousIngest {
         firstColFams[index] = cf;
         firstColQuals[index] = cq;
         
-        Mutation m = genMutation(rowLong, cf, cq, ingestInstanceId, count, null, r, checksum);
+        Mutation m = genMutation(rowLong, cf, cq, cv, ingestInstanceId, count, null, r, checksum);
         count++;
         bw.addMutation(m);
       }
@@ -171,7 +212,7 @@ public class ContinuousIngest {
           long rowLong = genLong(min, max, r);
           byte[] prevRow = genRow(prevRows[index]);
           prevRows[index] = rowLong;
-          Mutation m = genMutation(rowLong, r.nextInt(maxColF), r.nextInt(maxColQ), ingestInstanceId,
count, prevRow, r, checksum);
+          Mutation m = genMutation(rowLong, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId,
count, prevRow, r, checksum);
           count++;
           bw.addMutation(m);
         }
@@ -184,7 +225,8 @@ public class ContinuousIngest {
       // create one big linked list, this makes all of the first inserts
       // point to something
       for (int index = 0; index < flushInterval - 1; index++) {
-        Mutation m = genMutation(firstRows[index], firstColFams[index], firstColQuals[index],
ingestInstanceId, count, genRow(prevRows[index + 1]), r, checksum);
+        Mutation m = genMutation(firstRows[index], firstColFams[index], firstColQuals[index],
cv, ingestInstanceId, count, genRow(prevRows[index + 1]), r,
+            checksum);
         count++;
         bw.addMutation(m);
       }
@@ -195,7 +237,7 @@ public class ContinuousIngest {
     
     bw.close();
   }
-  
+
   private static long flush(BatchWriter bw, long count, final int flushInterval, long lastFlushTime)
throws MutationsRejectedException {
     long t1 = System.currentTimeMillis();
     bw.flush();
@@ -205,7 +247,8 @@ public class ContinuousIngest {
     return lastFlushTime;
   }
   
-  public static Mutation genMutation(long rowLong, int cfInt, int cqInt, byte[] ingestInstanceId,
long count, byte[] prevRow, Random r, boolean checksum) {
+  public static Mutation genMutation(long rowLong, int cfInt, int cqInt, ColumnVisibility
cv, byte[] ingestInstanceId, long count, byte[] prevRow, Random r,
+      boolean checksum) {
     // Adler32 is supposed to be faster, but according to wikipedia is not good for small
data.... so used CRC32 instead
     CRC32 cksum = null;
     
@@ -219,11 +262,12 @@ public class ContinuousIngest {
       cksum.update(rowString);
       cksum.update(cfString);
       cksum.update(cqString);
+      cksum.update(cv.getExpression());
     }
     
     Mutation m = new Mutation(new Text(rowString));
     
-    m.put(new Text(cfString), new Text(cqString), createValue(ingestInstanceId, count, prevRow,
cksum));
+    m.put(new Text(cfString), new Text(cqString), cv, createValue(ingestInstanceId, count,
prevRow, cksum));
     return m;
   }
   

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
Sat Oct 13 00:39:07 2012
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.Mut
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -64,6 +65,8 @@ public class ContinuousMoru extends Conf
     private byte[] iiId;
     private long count;
     
+    private static final ColumnVisibility EMPTY_VIS = new ColumnVisibility();
+
     public void setup(Context context) throws IOException, InterruptedException {
       int max_cf = context.getConfiguration().getInt(MAX_CF, -1);
       int max_cq = context.getConfiguration().getInt(MAX_CQ, -1);
@@ -92,7 +95,8 @@ public class ContinuousMoru extends Conf
         int offset = ContinuousWalk.getPrevRowOffset(val);
         if (offset > 0) {
           long rowLong = Long.parseLong(new String(val, offset, 16), 16);
-          Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq),
iiId, count++, key.getRowData().toArray(), random,
+          Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq),
EMPTY_VIS, iiId, count++, key.getRowData()
+              .toArray(), random,
               true);
           context.write(null, m);
         }

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
Sat Oct 13 00:39:07 2012
@@ -29,7 +29,9 @@ import org.apache.accumulo.core.client.Z
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.test.continuous.ContinuousWalk.RandomAuths;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Level;
@@ -38,6 +40,7 @@ import org.apache.log4j.PatternLayout;
 
 public class ContinuousScanner {
   private static String debugLog = null;
+  private static String authsFile = null;
   
   private static String[] processOptions(String[] args) {
     ArrayList<String> al = new ArrayList<String>();
@@ -45,6 +48,8 @@ public class ContinuousScanner {
     for (int i = 0; i < args.length; i++) {
       if (args[i].equals("--debug")) {
         debugLog = args[++i];
+      } else if (args[i].equals("--auths")) {
+        authsFile = args[++i];
       } else {
         al.add(args[i]);
       }
@@ -59,7 +64,7 @@ public class ContinuousScanner {
     
     if (args.length != 9) {
       throw new IllegalArgumentException("usage : " + ContinuousScanner.class.getName()
-          + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <min> <max> <sleep time> <num to scan>");
+          + " [--debug <debug log>] [--auths <file>] <instance name> <zookeepers>
<user> <pass> <table> <min> <max> <sleep time> <num
to scan>");
     }
     
     if (debugLog != null) {
@@ -69,6 +74,8 @@ public class ContinuousScanner {
       logger.addAppender(new FileAppender(new PatternLayout("%d{dd HH:mm:ss,SSS} [%-8c{2}]
%-5p: %m%n"), debugLog, true));
     }
     
+    Random r = new Random();
+
     String instanceName = args[0];
     String zooKeepers = args[1];
     
@@ -85,11 +92,12 @@ public class ContinuousScanner {
     
     int numToScan = Integer.parseInt(args[8]);
     
+    RandomAuths randomAuths = new RandomAuths(authsFile);
+
     Instance instance = new ZooKeeperInstance(instanceName, zooKeepers);
     Connector conn = instance.getConnector(user, password.getBytes());
-    Scanner scanner = conn.createScanner(table, Constants.NO_AUTHS);
-    
-    Random r = new Random();
+    Authorizations auths = randomAuths.getAuths(r);
+    Scanner scanner = conn.createScanner(table, auths);
     
     double delta = Math.min(.05, .05 / (numToScan / 1000.0));
     // System.out.println("Delta "+delta);

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
Sat Oct 13 00:39:07 2012
@@ -23,6 +23,7 @@ import java.util.HashSet;
 import java.util.Random;
 import java.util.Set;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
@@ -135,6 +136,20 @@ public class ContinuousVerify extends Co
   
   @Override
   public int run(String[] args) throws Exception {
+    
+    String auths = "";
+    ArrayList<String> argsList = new ArrayList<String>();
+    
+    for (int i = 0; i < args.length; i++) {
+      if (args[i].equals("--auths")) {
+        auths = args[++i];
+      } else {
+        argsList.add(args[i]);
+      }
+    }
+    
+    args = argsList.toArray(new String[0]);
+
     if (args.length != 9) {
       throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
           + " <instance name> <zookeepers> <user> <pass> <table>
<output dir> <max mappers> <num reducers> <scan offline>");
@@ -165,9 +180,16 @@ public class ContinuousVerify extends Co
     }
 
     job.setInputFormatClass(AccumuloInputFormat.class);
-    AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), clone,
new Authorizations());
+    Authorizations authorizations;
+    if (auths == null || auths.trim().equals(""))
+      authorizations = Constants.NO_AUTHS;
+    else
+      authorizations = new Authorizations(auths.split(","));
+
+    AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), clone,
authorizations);
     AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), instance, zookeepers);
     AccumuloInputFormat.setScanOffline(job.getConfiguration(), scanOffline);
+
     // set up ranges
     try {
       Set<Range> ranges = new ZooKeeperInstance(instance, zookeepers).getConnector(user,
pass.getBytes()).tableOperations()

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java
Sat Oct 13 00:39:07 2012
@@ -16,8 +16,13 @@
  */
 package org.apache.accumulo.server.test.continuous;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.InetAddress;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map.Entry;
 import java.util.Random;
 import java.util.zip.CRC32;
@@ -37,6 +42,9 @@ import org.apache.accumulo.core.data.Val
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.server.Accumulo;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Level;
@@ -47,6 +55,7 @@ import org.apache.log4j.PatternLayout;
 public class ContinuousWalk {
   
   private static String debugLog = null;
+  private static String authsFile = null;
   
   static class BadChecksumException extends RuntimeException {
     
@@ -64,6 +73,8 @@ public class ContinuousWalk {
     for (int i = 0; i < args.length; i++) {
       if (args[i].equals("--debug")) {
         debugLog = args[++i];
+      } else if (args[i].equals("--auths")) {
+        authsFile = args[++i];
       } else {
         al.add(args[i]);
       }
@@ -72,13 +83,41 @@ public class ContinuousWalk {
     return al.toArray(new String[al.size()]);
   }
   
+  static class RandomAuths {
+    private List<Authorizations> auths;
+    
+    RandomAuths(String file) throws IOException {
+      if (file == null) {
+        auths = Collections.singletonList(Constants.NO_AUTHS);
+        return;
+      }
+      
+      auths = new ArrayList<Authorizations>();
+      
+      FileSystem fs = FileSystem.get(new Configuration());
+      BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(file))));
+      
+      String line;
+      
+      while ((line = in.readLine()) != null) {
+        auths.add(new Authorizations(line.split(",")));
+      }
+      
+      in.close();
+    }
+    
+    Authorizations getAuths(Random r) {
+      return auths.get(r.nextInt(auths.size()));
+    }
+  }
+
   public static void main(String[] args) throws Exception {
     
     args = processOptions(args);
     
     if (args.length != 8) {
       throw new IllegalArgumentException("usage : " + ContinuousWalk.class.getName()
-          + " [--debug <debug log>] <instance name> <zookeepers> <user>
<pass> <table> <min> <max> <sleep time>");
+          + " [--debug <debug log>] [--auths <file>] <instance name> <zookeepers>
<user> <pass> <table> <min> <max> <sleep time>");
     }
     
     if (debugLog != null) {
@@ -108,13 +147,14 @@ public class ContinuousWalk {
     Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path, localhost, "cwalk",
1000));
     Accumulo.enableTracing(localhost, "ContinuousWalk");
     Connector conn = instance.getConnector(user, password.getBytes());
-    Scanner scanner = conn.createScanner(table, new Authorizations());
     
     Random r = new Random();
+    RandomAuths randomAuths = new RandomAuths(authsFile);
     
     ArrayList<Value> values = new ArrayList<Value>();
     
     while (true) {
+      Scanner scanner = conn.createScanner(table, randomAuths.getAuths(r));
       String row = findAStartRow(min, max, scanner, r);
       
       while (row != null) {
@@ -228,6 +268,7 @@ public class ContinuousWalk {
     cksum.update(key.getRowData().toArray());
     cksum.update(key.getColumnFamilyData().toArray());
     cksum.update(key.getColumnQualifierData().toArray());
+    cksum.update(key.getColumnVisibilityData().toArray());
     cksum.update(value.get(), 0, ckOff);
     
     if (cksum.getValue() != storedCksum) {

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java
Sat Oct 13 00:39:07 2012
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.client.C
 import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.server.test.continuous.ContinuousIngest;
 
 public class Ingest extends ScaleTest {
@@ -91,6 +92,8 @@ public class Ingest extends ScaleTest {
     long count = 0;
     long totalBytes = 0;
     
+    ColumnVisibility cv = new ColumnVisibility();
+
     // start timer
     startTimer();
     
@@ -98,7 +101,7 @@ public class Ingest extends ScaleTest {
     while (count < numIngestEntries) {
       count++;
       long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
-      Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ),
ingestInstanceId.getBytes(), count, null, r, false);
+      Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ),
cv, ingestInstanceId.getBytes(), count, null, r, false);
       totalBytes += m.numBytes();
       try {
         bw.addMutation(m);

Modified: accumulo/trunk/test/system/continuous/continuous-env.sh.example
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/continuous-env.sh.example?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/continuous-env.sh.example (original)
+++ accumulo/trunk/test/system/continuous/continuous-env.sh.example Sat Oct 13 00:39:07 2012
@@ -47,6 +47,11 @@ MAX_CF=32767
 #the maximum number of random column qualifiers to generate
 MAX_CQ=32767
 
+#an optional file in hdfs containing visibilites.  If left blank, then column
+#visibility will not be set.  If specified then a random line will be selected
+#from the file and used for column visibility for each linked list.
+VISIBILITIES=''
+
 #the max memory (in bytes) each ingester will use to buffer writes
 MAX_MEM=100000000
 
@@ -59,6 +64,11 @@ NUM_THREADS=4
 #the amount of time (in millis) to sleep between each query
 SLEEP_TIME=10
 
+#an optional file in hdfs containing line of comma seperated auths.  If
+#specified, walkers will randomly select lines from this file and use that to
+#set auths.
+AUTHS=''
+
 #time amount of time (in minutes) the agitator should sleep before killing
 KILL_SLEEP_TIME=20
 
@@ -81,6 +91,8 @@ VERFIY_OUT=/tmp/continuous_verify
 VERIFY_MAX_MAPS=64
 VERIFY_REDUCERS=64
 SCAN_OFFLINE=false
+#comma separated list of auths to use for verify
+VERIFY_AUTHS=''
 
 #settings related to the batch walker
 BATCH_WALKER_SLEEP=180000

Modified: accumulo/trunk/test/system/continuous/run-verify.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/run-verify.sh?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/run-verify.sh (original)
+++ accumulo/trunk/test/system/continuous/run-verify.sh Sat Oct 13 00:39:07 2012
@@ -18,5 +18,11 @@
 
 . mapred-setup.sh
 
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERFIY_OUT $VERIFY_MAX_MAPS
$VERIFY_REDUCERS $SCAN_OFFLINE
+AUTH_OPT="";
+
+if [ -n "$VERIFY_AUTHS" ] ; then
+	AUTH_OPT="--auths $VERIFY_AUTHS";
+fi
+
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERFIY_OUT
$VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
 

Modified: accumulo/trunk/test/system/continuous/start-batchwalkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-batchwalkers.sh?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-batchwalkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-batchwalkers.sh Sat Oct 13 00:39:07 2012
@@ -19,10 +19,16 @@
 . continuous-env.sh
 
 DEBUG_OPT="";
+AUTH_OPT="";
 
 if [ "$DEBUG_BATCH_WALKER" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.log";
 fi
 
-pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo
org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $INSTANCE_NAME
$ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS
>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
+if [ -n "$AUTHS" ] ; then
+	AUTH_OPT="--auths \"$AUTHS\"";
+fi
+
+
+pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo
org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $AUTH_OPT $INSTANCE_NAME
$ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS
>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-ingest.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-ingest.sh?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-ingest.sh (original)
+++ accumulo/trunk/test/system/continuous/start-ingest.sh Sat Oct 13 00:39:07 2012
@@ -19,10 +19,15 @@
 . continuous-env.sh
 
 DEBUG_OPT="";
+VIS_OPT="";
 
 if [ "$DEBUG_INGEST" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.log";
 fi
 
-pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM
$MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out
2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
+if [ -n "$VISIBILITIES" ] ; then
+	VIS_OPT="--visibilities \"$VISIBILITIES\"";
+fi
+
+pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest
$DEBUG_OPT $VIS_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF
$MAX_CQ $MAX_MEM $MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out
2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-scanners.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-scanners.sh?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-scanners.sh (original)
+++ accumulo/trunk/test/system/continuous/start-scanners.sh Sat Oct 13 00:39:07 2012
@@ -19,10 +19,16 @@
 . continuous-env.sh
 
 DEBUG_OPT="";
+AUTH_OPT="";
 
 if [ "$DEBUG_SCANNER" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.log";
 fi
 
-pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner
$DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME
$SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
+if [ -n "$AUTHS" ] ; then
+	AUTH_OPT="--auths \"$AUTHS\"";
+fi
+
+
+pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner
$DEBUG_OPT $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME
$SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
 

Modified: accumulo/trunk/test/system/continuous/start-walkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-walkers.sh?rev=1397746&r1=1397745&r2=1397746&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-walkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-walkers.sh Sat Oct 13 00:39:07 2012
@@ -19,10 +19,16 @@
 . continuous-env.sh
 
 DEBUG_OPT="";
+AUTH_OPT="";
 
 if [ "$DEBUG_WALKER" = "on" ] ; then
 	DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.log";
 fi
 
-pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk
$DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err
&" < /dev/null
+if [ -n "$AUTHS" ] ; then
+	AUTH_OPT="--auths \"$AUTHS\"";
+fi
+
+
+pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk
$DEBUG_OPT $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME
>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date
+%Y%m%d%H%M%S\`_\`hostname\`_walk.err &" < /dev/null
 



Mime
View raw message