Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C59EB9975 for ; Sat, 13 Oct 2012 01:08:49 +0000 (UTC) Received: (qmail 30297 invoked by uid 500); 13 Oct 2012 01:08:49 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 30260 invoked by uid 500); 13 Oct 2012 01:08:49 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 30251 invoked by uid 99); 13 Oct 2012 01:08:49 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Oct 2012 01:08:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Oct 2012 01:08:43 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 97CC723888E3 for ; Sat, 13 Oct 2012 01:07:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1397757 - in /accumulo/branches/1.4: ./ src/ src/core/ src/server/ src/server/src/main/java/org/apache/accumulo/server/test/continuous/ src/server/src/main/java/org/apache/accumulo/server/test/scalability/ test/system/continuous/ Date: Sat, 13 Oct 2012 01:07:57 -0000 To: commits@accumulo.apache.org From: kturner@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121013010758.97CC723888E3@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kturner Date: Sat Oct 13 01:07:57 2012 New Revision: 1397757 URL: http://svn.apache.org/viewvc?rev=1397757&view=rev Log: ACCUMULO-810 added authorizations to continuous ingest test (merged from trunk) Modified: accumulo/branches/1.4/ (props changed) accumulo/branches/1.4/src/ (props changed) accumulo/branches/1.4/src/core/ (props changed) accumulo/branches/1.4/src/server/ (props changed) accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example accumulo/branches/1.4/test/system/continuous/run-verify.sh accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh accumulo/branches/1.4/test/system/continuous/start-ingest.sh accumulo/branches/1.4/test/system/continuous/start-scanners.sh accumulo/branches/1.4/test/system/continuous/start-walkers.sh Propchange: accumulo/branches/1.4/ ------------------------------------------------------------------------------ Merged /accumulo/trunk:r1397746 Propchange: accumulo/branches/1.4/src/ ------------------------------------------------------------------------------ Merged /accumulo/trunk/src:r1397746 Merged /accumulo/trunk:r1397746 Propchange: accumulo/branches/1.4/src/core/ ------------------------------------------------------------------------------ Merged /accumulo/trunk/src/core:r1397746 Merged /accumulo/trunk/core:r1397746 Propchange: accumulo/branches/1.4/src/server/ ------------------------------------------------------------------------------ Merged /accumulo/trunk/src/server:r1397746 Merged /accumulo/trunk/server:r1397746 Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java Sat Oct 13 01:07:57 2012 @@ -32,7 +32,9 @@ import org.apache.accumulo.core.client.Z import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.server.test.continuous.ContinuousWalk.RandomAuths; import org.apache.hadoop.io.Text; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; @@ -41,6 +43,7 @@ import org.apache.log4j.PatternLayout; public class ContinuousBatchWalker { private static String debugLog = null; + private static String authsFile = null; private static String[] processOptions(String[] args) { ArrayList al = new ArrayList(); @@ -48,6 +51,8 @@ public class ContinuousBatchWalker { for (int i = 0; i < args.length; i++) { if (args[i].equals("--debug")) { debugLog = args[++i]; + } else if (args[i].equals("--auths")) { + authsFile = args[++i]; } else { al.add(args[i]); } @@ -62,7 +67,7 @@ public class ContinuousBatchWalker { if (args.length != 10) { throw new IllegalArgumentException("usage : " + ContinuousBatchWalker.class.getName() - + " [--debug ] "); + + " [--debug ] [--auths ]
"); } if (debugLog != null) { @@ -88,12 +93,14 @@ public class ContinuousBatchWalker { int batchSize = Integer.parseInt(args[8]); int numQueryThreads = Integer.parseInt(args[9]); - Connector conn = new ZooKeeperInstance(instanceName, zooKeepers).getConnector(user, password.getBytes()); - Scanner scanner = conn.createScanner(table, Constants.NO_AUTHS); - BatchScanner bs = conn.createBatchScanner(table, Constants.NO_AUTHS, numQueryThreads); - Random r = new Random(); - + RandomAuths randomAuths = new RandomAuths(authsFile); + Authorizations auths = randomAuths.getAuths(r); + + Connector conn = new ZooKeeperInstance(instanceName, zooKeepers).getConnector(user, password.getBytes()); + Scanner scanner = conn.createScanner(table, auths); + BatchScanner bs = conn.createBatchScanner(table, auths, numQueryThreads); + while (true) { Set batch = getBatch(scanner, min, max, batchSize, r); Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java Sat Oct 13 01:07:57 2012 @@ -16,8 +16,12 @@ */ package org.apache.accumulo.server.test.continuous; +import java.io.BufferedReader; +import java.io.InputStreamReader; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Random; import java.util.UUID; import java.util.zip.CRC32; @@ -36,8 +40,12 @@ import org.apache.accumulo.core.client.T import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.server.test.FastFormat; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; @@ -47,15 +55,44 @@ import org.apache.log4j.PatternLayout; public class ContinuousIngest { + private static String visFile = null; private static String debugLog = null; private static final byte[] EMPTY_BYTES = new byte[0]; + private static List visibilities; + + private static void initVisibilities() throws Exception { + if (visFile == null) { + visibilities = Collections.singletonList(new ColumnVisibility()); + return; + } + + visibilities = new ArrayList(); + + FileSystem fs = FileSystem.get(new Configuration()); + BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(visFile)))); + + String line; + + while ((line = in.readLine()) != null) { + visibilities.add(new ColumnVisibility(line)); + } + + in.close(); + } + + private static ColumnVisibility getVisibility(Random rand) { + return visibilities.get(rand.nextInt(visibilities.size())); + } + private static String[] processOptions(String[] args) { ArrayList al = new ArrayList(); for (int i = 0; i < args.length; i++) { if (args[i].equals("--debug")) { debugLog = args[++i]; + } else if (args[i].equals("--visibilities")) { + visFile = args[++i]; } else { al.add(args[i]); } @@ -72,7 +109,7 @@ public class ContinuousIngest { throw new IllegalArgumentException( "usage : " + ContinuousIngest.class.getName() - + " [--debug ]
"); + + " [--debug ] [--visibilities ]
"); } if (debugLog != null) { @@ -82,6 +119,8 @@ public class ContinuousIngest { logger.addAppender(new FileAppender(new PatternLayout("%d{dd HH:mm:ss,SSS} [%-8c{2}] %-5p: %m%n"), debugLog, true)); } + initVisibilities(); + String instanceName = args[0]; String zooKeepers = args[1]; @@ -142,6 +181,8 @@ public class ContinuousIngest { out: while (true) { // generate first set of nodes + ColumnVisibility cv = getVisibility(r); + for (int index = 0; index < flushInterval; index++) { long rowLong = genLong(min, max, r); prevRows[index] = rowLong; @@ -153,7 +194,7 @@ public class ContinuousIngest { firstColFams[index] = cf; firstColQuals[index] = cq; - Mutation m = genMutation(rowLong, cf, cq, ingestInstanceId, count, null, r, checksum); + Mutation m = genMutation(rowLong, cf, cq, cv, ingestInstanceId, count, null, r, checksum); count++; bw.addMutation(m); } @@ -168,7 +209,7 @@ public class ContinuousIngest { long rowLong = genLong(min, max, r); byte[] prevRow = genRow(prevRows[index]); prevRows[index] = rowLong; - Mutation m = genMutation(rowLong, r.nextInt(maxColF), r.nextInt(maxColQ), ingestInstanceId, count, prevRow, r, checksum); + Mutation m = genMutation(rowLong, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId, count, prevRow, r, checksum); count++; bw.addMutation(m); } @@ -181,7 +222,8 @@ public class ContinuousIngest { // create one big linked list, this makes all of the first inserts // point to something for (int index = 0; index < flushInterval - 1; index++) { - Mutation m = genMutation(firstRows[index], firstColFams[index], firstColQuals[index], ingestInstanceId, count, genRow(prevRows[index + 1]), r, checksum); + Mutation m = genMutation(firstRows[index], firstColFams[index], firstColQuals[index], cv, ingestInstanceId, count, genRow(prevRows[index + 1]), r, + checksum); count++; bw.addMutation(m); } @@ -192,7 +234,7 @@ public class ContinuousIngest { bw.close(); } - + private static long flush(BatchWriter bw, long count, final int flushInterval, long lastFlushTime) throws MutationsRejectedException { long t1 = System.currentTimeMillis(); bw.flush(); @@ -202,7 +244,8 @@ public class ContinuousIngest { return lastFlushTime; } - public static Mutation genMutation(long rowLong, int cfInt, int cqInt, byte[] ingestInstanceId, long count, byte[] prevRow, Random r, boolean checksum) { + public static Mutation genMutation(long rowLong, int cfInt, int cqInt, ColumnVisibility cv, byte[] ingestInstanceId, long count, byte[] prevRow, Random r, + boolean checksum) { // Adler32 is supposed to be faster, but according to wikipedia is not good for small data.... so used CRC32 instead CRC32 cksum = null; @@ -216,11 +259,12 @@ public class ContinuousIngest { cksum.update(rowString); cksum.update(cfString); cksum.update(cqString); + cksum.update(cv.getExpression()); } Mutation m = new Mutation(new Text(rowString)); - m.put(new Text(cfString), new Text(cqString), createValue(ingestInstanceId, count, prevRow, cksum)); + m.put(new Text(cfString), new Text(cqString), cv, createValue(ingestInstanceId, count, prevRow, cksum)); return m; } Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java Sat Oct 13 01:07:57 2012 @@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.Mut import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -64,6 +65,8 @@ public class ContinuousMoru extends Conf private byte[] iiId; private long count; + private static final ColumnVisibility EMPTY_VIS = new ColumnVisibility(); + public void setup(Context context) throws IOException, InterruptedException { int max_cf = context.getConfiguration().getInt(MAX_CF, -1); int max_cq = context.getConfiguration().getInt(MAX_CQ, -1); @@ -92,7 +95,8 @@ public class ContinuousMoru extends Conf int offset = ContinuousWalk.getPrevRowOffset(val); if (offset > 0) { long rowLong = Long.parseLong(new String(val, offset, 16), 16); - Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq), iiId, count++, key.getRowData().toArray(), random, + Mutation m = ContinuousIngest.genMutation(rowLong, random.nextInt(max_cf), random.nextInt(max_cq), EMPTY_VIS, iiId, count++, key.getRowData() + .toArray(), random, true); context.write(null, m); } Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java Sat Oct 13 01:07:57 2012 @@ -29,7 +29,9 @@ import org.apache.accumulo.core.client.Z import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.server.test.continuous.ContinuousWalk.RandomAuths; import org.apache.hadoop.io.Text; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; @@ -38,6 +40,7 @@ import org.apache.log4j.PatternLayout; public class ContinuousScanner { private static String debugLog = null; + private static String authsFile = null; private static String[] processOptions(String[] args) { ArrayList al = new ArrayList(); @@ -45,6 +48,8 @@ public class ContinuousScanner { for (int i = 0; i < args.length; i++) { if (args[i].equals("--debug")) { debugLog = args[++i]; + } else if (args[i].equals("--auths")) { + authsFile = args[++i]; } else { al.add(args[i]); } @@ -59,7 +64,7 @@ public class ContinuousScanner { if (args.length != 9) { throw new IllegalArgumentException("usage : " + ContinuousScanner.class.getName() - + " [--debug ]
"); + + " [--debug ] [--auths ]
"); } if (debugLog != null) { @@ -69,6 +74,8 @@ public class ContinuousScanner { logger.addAppender(new FileAppender(new PatternLayout("%d{dd HH:mm:ss,SSS} [%-8c{2}] %-5p: %m%n"), debugLog, true)); } + Random r = new Random(); + String instanceName = args[0]; String zooKeepers = args[1]; @@ -85,11 +92,12 @@ public class ContinuousScanner { int numToScan = Integer.parseInt(args[8]); + RandomAuths randomAuths = new RandomAuths(authsFile); + Instance instance = new ZooKeeperInstance(instanceName, zooKeepers); Connector conn = instance.getConnector(user, password.getBytes()); - Scanner scanner = conn.createScanner(table, Constants.NO_AUTHS); - - Random r = new Random(); + Authorizations auths = randomAuths.getAuths(r); + Scanner scanner = conn.createScanner(table, auths); double delta = Math.min(.05, .05 / (numToScan / 1000.0)); // System.out.println("Delta "+delta); Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java Sat Oct 13 01:07:57 2012 @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Random; import java.util.Set; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; @@ -135,6 +136,20 @@ public class ContinuousVerify extends Co @Override public int run(String[] args) throws Exception { + + String auths = ""; + ArrayList argsList = new ArrayList(); + + for (int i = 0; i < args.length; i++) { + if (args[i].equals("--auths")) { + auths = args[++i]; + } else { + argsList.add(args[i]); + } + } + + args = argsList.toArray(new String[0]); + if (args.length != 9) { throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName() + "
"); @@ -165,9 +180,16 @@ public class ContinuousVerify extends Co } job.setInputFormatClass(AccumuloInputFormat.class); - AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), clone, new Authorizations()); + Authorizations authorizations; + if (auths == null || auths.trim().equals("")) + authorizations = Constants.NO_AUTHS; + else + authorizations = new Authorizations(auths.split(",")); + + AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), clone, authorizations); AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), instance, zookeepers); AccumuloInputFormat.setScanOffline(job.getConfiguration(), scanOffline); + // set up ranges try { Set ranges = new ZooKeeperInstance(instance, zookeepers).getConnector(user, pass.getBytes()).tableOperations() Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java Sat Oct 13 01:07:57 2012 @@ -16,8 +16,13 @@ */ package org.apache.accumulo.server.test.continuous; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map.Entry; import java.util.Random; import java.util.zip.CRC32; @@ -37,6 +42,9 @@ import org.apache.accumulo.core.data.Val import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.server.Accumulo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; @@ -47,6 +55,7 @@ import org.apache.log4j.PatternLayout; public class ContinuousWalk { private static String debugLog = null; + private static String authsFile = null; static class BadChecksumException extends RuntimeException { @@ -64,6 +73,8 @@ public class ContinuousWalk { for (int i = 0; i < args.length; i++) { if (args[i].equals("--debug")) { debugLog = args[++i]; + } else if (args[i].equals("--auths")) { + authsFile = args[++i]; } else { al.add(args[i]); } @@ -72,13 +83,41 @@ public class ContinuousWalk { return al.toArray(new String[al.size()]); } + static class RandomAuths { + private List auths; + + RandomAuths(String file) throws IOException { + if (file == null) { + auths = Collections.singletonList(Constants.NO_AUTHS); + return; + } + + auths = new ArrayList(); + + FileSystem fs = FileSystem.get(new Configuration()); + BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(file)))); + + String line; + + while ((line = in.readLine()) != null) { + auths.add(new Authorizations(line.split(","))); + } + + in.close(); + } + + Authorizations getAuths(Random r) { + return auths.get(r.nextInt(auths.size())); + } + } + public static void main(String[] args) throws Exception { args = processOptions(args); if (args.length != 8) { throw new IllegalArgumentException("usage : " + ContinuousWalk.class.getName() - + " [--debug ]
"); + + " [--debug ] [--auths ]
"); } if (debugLog != null) { @@ -108,13 +147,14 @@ public class ContinuousWalk { Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path, localhost, "cwalk", 1000)); Accumulo.enableTracing(localhost, "ContinuousWalk"); Connector conn = instance.getConnector(user, password.getBytes()); - Scanner scanner = conn.createScanner(table, new Authorizations()); Random r = new Random(); + RandomAuths randomAuths = new RandomAuths(authsFile); ArrayList values = new ArrayList(); while (true) { + Scanner scanner = conn.createScanner(table, randomAuths.getAuths(r)); String row = findAStartRow(min, max, scanner, r); while (row != null) { @@ -228,6 +268,7 @@ public class ContinuousWalk { cksum.update(key.getRowData().toArray()); cksum.update(key.getColumnFamilyData().toArray()); cksum.update(key.getColumnQualifierData().toArray()); + cksum.update(key.getColumnVisibilityData().toArray()); cksum.update(value.get(), 0, ckOff); if (cksum.getValue() != storedCksum) { Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java (original) +++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java Sat Oct 13 01:07:57 2012 @@ -24,6 +24,7 @@ import org.apache.accumulo.core.client.C import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.server.test.continuous.ContinuousIngest; public class Ingest extends ScaleTest { @@ -88,6 +89,8 @@ public class Ingest extends ScaleTest { long count = 0; long totalBytes = 0; + ColumnVisibility cv = new ColumnVisibility(); + // start timer startTimer(); @@ -95,7 +98,7 @@ public class Ingest extends ScaleTest { while (count < numIngestEntries) { count++; long rowId = ContinuousIngest.genLong(minRow, maxRow, r); - Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), ingestInstanceId.getBytes(), count, null, r, false); + Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId.getBytes(), count, null, r, false); totalBytes += m.numBytes(); try { bw.addMutation(m); Modified: accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example (original) +++ accumulo/branches/1.4/test/system/continuous/continuous-env.sh.example Sat Oct 13 01:07:57 2012 @@ -47,6 +47,11 @@ MAX_CF=32767 #the maximum number of random column qualifiers to generate MAX_CQ=32767 +#an optional file in hdfs containing visibilites. If left blank, then column +#visibility will not be set. If specified then a random line will be selected +#from the file and used for column visibility for each linked list. +VISIBILITIES='' + #the max memory (in bytes) each ingester will use to buffer writes MAX_MEM=100000000 @@ -59,6 +64,11 @@ NUM_THREADS=4 #the amount of time (in millis) to sleep between each query SLEEP_TIME=10 +#an optional file in hdfs containing line of comma seperated auths. If +#specified, walkers will randomly select lines from this file and use that to +#set auths. +AUTHS='' + #time amount of time (in minutes) the agitator should sleep before killing KILL_SLEEP_TIME=20 @@ -81,6 +91,8 @@ VERFIY_OUT=/tmp/continuous_verify VERIFY_MAX_MAPS=64 VERIFY_REDUCERS=64 SCAN_OFFLINE=false +#comma separated list of auths to use for verify +VERIFY_AUTHS='' #settings related to the batch walker BATCH_WALKER_SLEEP=180000 Modified: accumulo/branches/1.4/test/system/continuous/run-verify.sh URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/run-verify.sh?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/run-verify.sh (original) +++ accumulo/branches/1.4/test/system/continuous/run-verify.sh Sat Oct 13 01:07:57 2012 @@ -18,5 +18,11 @@ . mapred-setup.sh -$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERFIY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE +AUTH_OPT=""; + +if [ -n "$VERIFY_AUTHS" ] ; then + AUTH_OPT="--auths $VERIFY_AUTHS"; +fi + +$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERFIY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE Modified: accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh (original) +++ accumulo/branches/1.4/test/system/continuous/start-batchwalkers.sh Sat Oct 13 01:07:57 2012 @@ -19,10 +19,16 @@ . continuous-env.sh DEBUG_OPT=""; +AUTH_OPT=""; if [ "$DEBUG_BATCH_WALKER" = "on" ] ; then DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.log"; fi -pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null +if [ -n "$AUTHS" ] ; then + AUTH_OPT="--auths \"$AUTHS\""; +fi + + +pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null Modified: accumulo/branches/1.4/test/system/continuous/start-ingest.sh URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/start-ingest.sh?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/start-ingest.sh (original) +++ accumulo/branches/1.4/test/system/continuous/start-ingest.sh Sat Oct 13 01:07:57 2012 @@ -19,6 +19,7 @@ . continuous-env.sh DEBUG_OPT=""; +VIS_OPT=""; if [ "$DEBUG_INGEST" = "on" ] ; then DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.log"; @@ -29,5 +30,9 @@ if [ -z $NUM ]; then NUM=9223372036854775807 fi -pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM $MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null +if [ -n "$VISIBILITIES" ] ; then + VIS_OPT="--visibilities \"$VISIBILITIES\""; +fi + +pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest $DEBUG_OPT $VIS_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM $MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null Modified: accumulo/branches/1.4/test/system/continuous/start-scanners.sh URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/start-scanners.sh?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/start-scanners.sh (original) +++ accumulo/branches/1.4/test/system/continuous/start-scanners.sh Sat Oct 13 01:07:57 2012 @@ -19,10 +19,16 @@ . continuous-env.sh DEBUG_OPT=""; +AUTH_OPT=""; if [ "$DEBUG_SCANNER" = "on" ] ; then DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.log"; fi -pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner $DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME $SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null +if [ -n "$AUTHS" ] ; then + AUTH_OPT="--auths \"$AUTHS\""; +fi + + +pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner $DEBUG_OPT $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME $SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null Modified: accumulo/branches/1.4/test/system/continuous/start-walkers.sh URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/continuous/start-walkers.sh?rev=1397757&r1=1397756&r2=1397757&view=diff ============================================================================== --- accumulo/branches/1.4/test/system/continuous/start-walkers.sh (original) +++ accumulo/branches/1.4/test/system/continuous/start-walkers.sh Sat Oct 13 01:07:57 2012 @@ -19,10 +19,16 @@ . continuous-env.sh DEBUG_OPT=""; +AUTH_OPT=""; if [ "$DEBUG_WALKER" = "on" ] ; then DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.log"; fi -pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err &" < /dev/null +if [ -n "$AUTHS" ] ; then + AUTH_OPT="--auths \"$AUTHS\""; +fi + + +pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk $DEBUG_OPT $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err &" < /dev/null