accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [2/4] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Date Mon, 25 Nov 2013 20:25:17 GMT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT

Conflicts:
	examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
	test/system/auto/simple/examples.py

In addition to merging from 1.4 also fixed a lot of issues with examples functional test that
were not issues in 1.4.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c175bd31
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c175bd31
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c175bd31

Branch: refs/heads/master
Commit: c175bd31cfb18a081e328ccf5b172f67fcea7491
Parents: 17ac692 1fe5f7f
Author: Keith Turner <kturner@apache.org>
Authored: Mon Nov 25 13:14:32 2013 -0500
Committer: Keith Turner <kturner@apache.org>
Committed: Mon Nov 25 13:14:32 2013 -0500

----------------------------------------------------------------------
 .../simple/client/RandomBatchScanner.java       |  21 ++--
 .../examples/simple/mapreduce/TableToFile.java  |   2 +-
 test/system/auto/simple/examples.py             | 103 ++++++++++---------
 3 files changed, 73 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 5194b67,0000000..2ae82b4
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,225 -1,0 +1,234 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.examples.simple.client;
 +
 +import java.util.Arrays;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +import java.util.concurrent.TimeUnit;
 +import java.util.Random;
 +
 +import org.apache.accumulo.core.cli.BatchScannerOpts;
 +import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +/**
 + * Internal class used to verify validity of data read.
 + */
 +class CountingVerifyingReceiver {
 +  private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
 +  
 +  long count = 0;
 +  int expectedValueSize = 0;
 +  HashMap<Text,Boolean> expectedRows;
 +  
 +  CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize)
{
 +    this.expectedRows = expectedRows;
 +    this.expectedValueSize = expectedValueSize;
 +  }
 +  
 +  public void receive(Key key, Value value) {
 +    
 +    String row = key.getRow().toString();
 +    long rowid = Integer.parseInt(row.split("_")[1]);
 +    
 +    byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);
 +    
 +    if (!Arrays.equals(expectedValue, value.get())) {
 +      log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue)
+ " got : " + new String(value.get()));
 +    }
 +    
 +    if (!expectedRows.containsKey(key.getRow())) {
 +      log.error("Got unexpected key " + key);
 +    } else {
 +      expectedRows.put(key.getRow(), true);
 +    }
 +    
 +    count++;
 +  }
 +}
 +
 +/**
 + * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch
for instructions.
 + */
 +public class RandomBatchScanner {
 +  private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
 +  
 +  /**
 +   * Generate a number of ranges, each covering a single random row.
 +   * 
 +   * @param num
 +   *          the number of ranges to generate
 +   * @param min
 +   *          the minimum row that will be generated
 +   * @param max
 +   *          the maximum row that will be generated
 +   * @param r
 +   *          a random number generator
 +   * @param ranges
 +   *          a set in which to store the generated ranges
 +   * @param expectedRows
 +   *          a map in which to store the rows covered by the ranges (initially mapped to
false)
 +   */
 +  static void generateRandomQueries(int num, long min, long max, Random r, HashSet<Range>
ranges, HashMap<Text,Boolean> expectedRows) {
 +    log.info(String.format("Generating %,d random queries...", num));
 +    while (ranges.size() < num) {
 +      long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
 +      
 +      Text row1 = new Text(String.format("row_%010d", rowid));
 +      
 +      Range range = new Range(new Text(row1));
 +      ranges.add(range);
 +      expectedRows.put(row1, false);
 +    }
 +    
 +    log.info("finished");
 +  }
 +  
 +  /**
 +   * Prints a count of the number of rows mapped to false.
 +   * 
 +   * @param expectedRows
++   * @return boolean indicating "were all the rows found?"
 +   */
-   private static void printRowsNotFound(HashMap<Text,Boolean> expectedRows) {
++  private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
 +    int count = 0;
++    boolean allFound = true;
 +    for (Entry<Text,Boolean> entry : expectedRows.entrySet())
 +      if (!entry.getValue())
 +        count++;
 +    
-     if (count > 0)
++    if (count > 0) {
 +      log.warn("Did not find " + count + " rows");
++      allFound = false;
++    }
++    return allFound;
 +  }
 +  
 +  /**
 +   * Generates a number of random queries, verifies that the key/value pairs returned were
in the queried ranges and that the values were generated by
 +   * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results.
 +   * 
 +   * @param num
 +   *          the number of queries to generate
 +   * @param min
 +   *          the min row to query
 +   * @param max
 +   *          the max row to query
 +   * @param evs
 +   *          the expected size of the values
 +   * @param r
 +   *          a random number generator
 +   * @param tsbr
 +   *          a batch scanner
++   * @return boolean indicating "did the queries go fine?"
 +   */
-   static void doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner
tsbr) {
++  static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner
tsbr) {
 +    
 +    HashSet<Range> ranges = new HashSet<Range>(num);
 +    HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
 +    
 +    generateRandomQueries(num, min, max, r, ranges, expectedRows);
 +    
 +    tsbr.setRanges(ranges);
 +    
 +    CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs);
 +    
 +    long t1 = System.currentTimeMillis();
 +    
 +    for (Entry<Key,Value> entry : tsbr) {
 +      receiver.receive(entry.getKey(), entry.getValue());
 +    }
 +    
 +    long t2 = System.currentTimeMillis();
 +    
 +    log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0),
((t2 - t1) / 1000.0)));
 +    log.info(String.format("num results : %,d%n", receiver.count));
 +    
-     printRowsNotFound(expectedRows);
++    return checkAllRowsFound(expectedRows);
 +  }
 +  
 +  public static class Opts  extends ClientOnRequiredTable {
 +    @Parameter(names="--min", description="miniumum row that will be generated")
 +    long min = 0;
 +    @Parameter(names="--max", description="maximum ow that will be generated")
 +    long max = 0;
 +    @Parameter(names="--num", required=true, description="number of ranges to generate")
 +    int num = 0;
 +    @Parameter(names="--size", required=true, description="size of the value to write")
 +    int size = 0;
 +    @Parameter(names="--seed", description="seed for pseudo-random number generator")
 +    Long seed = null;
 +  }
 +  
 +  /**
 +   * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}.
Completes scans twice to compare times for a fresh query with those for
 +   * a repeated query which has cached metadata and connections already established.
 +   * 
 +   * @param args
 +   * @throws AccumuloException
 +   * @throws AccumuloSecurityException
 +   * @throws TableNotFoundException
 +   */
 +  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException,
TableNotFoundException {
 +    Opts opts = new Opts();
 +    BatchScannerOpts bsOpts = new BatchScannerOpts();
 +    opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
 +    
 +    Connector connector = opts.getConnector();
 +    BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths,
bsOpts.scanThreads);
 +    batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
 +    
 +    Random r;
 +    if (opts.seed == null)
 +      r = new Random();
 +    else
 +      r = new Random(opts.seed);
 +    
 +    // do one cold
-     doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++    boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
 +    
 +    System.gc();
 +    System.gc();
 +    System.gc();
 +    
 +    if (opts.seed == null)
 +      r = new Random();
 +    else
 +      r = new Random(opts.seed);
 +    
 +    // do one hot (connections already established, metadata table cached)
-     doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++    status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size,
r, batchReader);
 +    
 +    batchReader.close();
++    if (!status) {
++      System.exit(1);
++    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index 4951852,0000000..d8eedef
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,130 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.examples.simple.mapreduce;
 +
 +import java.io.IOException;
 +import java.util.HashSet;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.format.DefaultFormatter;
 +import org.apache.hadoop.conf.Configured;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.NullWritable;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.mapreduce.Job;
 +import org.apache.hadoop.mapreduce.Mapper;
 +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 +import org.apache.hadoop.util.Tool;
 +import org.apache.hadoop.util.ToolRunner;
 +
 +import com.beust.jcommander.Parameter;
 +
 +/**
 + * Takes a table and outputs the specified column to a set of part files on hdfs accumulo
accumulo.examples.mapreduce.TableToFile <username> <password>
 + * <tablename> <column> <hdfs-output-path>
 + */
 +public class TableToFile extends Configured implements Tool {
 +  
 +  static class Opts extends ClientOnRequiredTable {
 +    @Parameter(names = "--output", description = "output directory", required = true)
 +    String output;
 +    @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...}
form")
-     String columns;
++    String columns = "";
 +  }
 +  
 +  /**
 +   * The Mapper class that given a row number, will generate the appropriate output line.
 +   */
 +  public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
 +    @Override
 +    public void map(Key row, Value data, Context context) throws IOException, InterruptedException
{
 +      final Key r = row;
 +      final Value v = data;
 +      Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
 +        @Override
 +        public Key getKey() {
 +          return r;
 +        }
 +        
 +        @Override
 +        public Value getValue() {
 +          return v;
 +        }
 +        
 +        @Override
 +        public Value setValue(Value value) {
 +          return null;
 +        }
 +      };
 +      context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
 +      context.setStatus("Outputed Value");
 +    }
 +  }
 +  
 +  @Override
 +  public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException,
AccumuloSecurityException {
 +    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
 +    job.setJarByClass(this.getClass());
 +    Opts opts = new Opts();
 +    opts.parseArgs(getClass().getName(), args);
 +    
 +    job.setInputFormatClass(AccumuloInputFormat.class);
 +    opts.setAccumuloConfigs(job);
 +    
 +    HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
 +    for (String col : opts.columns.split(",")) {
 +      int idx = col.indexOf(":");
 +      Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
 +      Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
 +      if (cf.getLength() > 0)
 +        columnsToFetch.add(new Pair<Text,Text>(cf, cq));
 +    }
 +    if (!columnsToFetch.isEmpty())
 +      AccumuloInputFormat.fetchColumns(job, columnsToFetch);
 +    
 +    job.setMapperClass(TTFMapper.class);
 +    job.setMapOutputKeyClass(NullWritable.class);
 +    job.setMapOutputValueClass(Text.class);
 +    
 +    job.setNumReduceTasks(0);
 +    
 +    job.setOutputFormatClass(TextOutputFormat.class);
 +    TextOutputFormat.setOutputPath(job, new Path(opts.output));
 +    
 +    job.waitForCompletion(true);
 +    return job.isSuccessful() ? 0 : 1;
 +  }
 +  
 +  /**
 +   * 
 +   * @param args
 +   *          instanceName zookeepers username password table columns outputpath
 +   * @throws Exception
 +   */
 +  public static void main(String[] args) throws Exception {
 +    int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
 +    if (res != 0)
 +      System.exit(res);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/test/system/auto/simple/examples.py
----------------------------------------------------------------------
diff --cc test/system/auto/simple/examples.py
index 069148d,b22411a..3ba425e
--- a/test/system/auto/simple/examples.py
+++ b/test/system/auto/simple/examples.py
@@@ -41,11 -41,15 +41,11 @@@ class Examples(TestUtilsMixin, unittest
      order = 21
  
      def runExample(self, cmd):
-         return self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
+         self.assert_(self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)),
"example exited with error status.")
  
 -    def ignoreExample(self, cmd):
 -        self.comment("  Ignoring results of command.")
 -        self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
 -
 -    def ashell(self, input):
 +    def ashell(self, input, expected = 0):
          out, err, code = self.shell(self.masterHost(), input + '\n')
 -        self.assert_(code == 0)
 +        self.assert_(code == expected)
          return out
  
      def comment(self, description):
@@@ -58,17 -68,9 +64,16 @@@
          self.wait(self.runOn('localhost', cmd))
  
      def runTest(self):
 +        examplesJar = glob.glob(ACCUMULO_HOME+'/lib/accumulo-examples-simple*.jar')[0]
- 
 +	self.comment("Testing MaxMutation constraint")
 +	self.ashell('createtable test_ingest\n'
 +                    'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n')
 +        handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest',
'-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD])
 +        out, err = handle.communicate()
 +        self.failIf(handle.returncode==0)
 +        self.failUnless(err.find("MutationsRejectedException: # constraint violations :
1") >= 0, "Was able to insert a mutation larger than max size")
 +        
          self.ashell('createtable %s\nsetauths -u %s -s A,B\nquit\n' %(table, ROOT))
 -        examplesJar = glob.glob(ACCUMULO_HOME+'/lib/examples-simple*.jar')[0]
 -
          self.comment("Testing dirlist example (a little)")
          self.comment("  ingesting accumulo source")
          self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.dirlist.Ingest',
@@@ -103,156 -105,167 +108,161 @@@
  
          self.comment("Testing bloom filters are fast for missing data")
          self.ashell('createtable bloom_test\nconfig -t bloom_test -s table.bloom.enabled=true\n')
 -        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter',
'-s', '7',
 -                     INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
 -                     '1000000', '0', '1000000000', '50', '2000000', '60000', '3', 'A')
 +        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter',
'--seed', '7',
 +                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
'-t', 'bloom_test',
 +                     '--num', '1000000', '--min', '0', '--max', '1000000000', '--size',
'50', '--batchMemory', '2M', '--batchLatency', '60s', 
 +                     '--batchThreads', '3')
          self.ashell('flush -t bloom_test -w\n')
          now = time.time()
 -        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner',
'-s', '7',
 -                     INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
 -                     500, 0, 1000000000, 50, 20, 'A')
 +        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner',
'--seed', '7',
 +                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
'-t', 'bloom_test',
 +                     '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50',
'--scanThreads', 4)
          diff = time.time() - now
          now = time.time()
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner',
'--seed', '8',
 -        self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner',
'-s', '8',
 -                     INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
 -                     500, 0, 1000000000, 50, 20, 'A')
++        self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner',
'--seed', '8',
 +                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
'-t', 'bloom_test',
 +                     '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50',
'--scanThreads', 4)
          diff2 = time.time() - now
          self.assert_(diff2 < diff)
  
          self.comment("Creating a sharded index of the accumulo java files")
          self.ashell('createtable shard\ncreatetable doc2term\nquit\n')
          self.execute('/bin/sh', '-c',
-                      'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.simple.examples.shard.Index
-i %s -z %s -t shard -u %s -p %s --partitions 30' %
 -                     'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index
%s %s shard %s %s 30' %
++                     'find server -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index
-i %s -z %s -t shard -u %s -p %s --partitions 30' %
                       (INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD))
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Query',
+         self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Query',
 -                     INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD,
 +                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '-u', ROOT, '-p',
ROOT_PASSWORD,
                       'foo', 'bar')
          self.comment("Creating a word index of the sharded files")
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Reverse',
-                      '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term',
'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
+         self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Reverse',
 -                     INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD)
++                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term',
'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
          self.comment("Making 1000 conjunctive queries of 5 random words")
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.ContinuousQuery',
-                      '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term',
'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--term', 5, '--count', 1000)
- 
-         self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt",
"/tmp/failures")
-         self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
+         self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.ContinuousQuery',
 -                     INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD,
5, 1000)
 -
 -        self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt",
"/tmp/failures")
 -        self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
++                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term',
'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--terms', 5, '--count', 1000)
++        self.executeIgnoreFail('hadoop', 'fs', '-rmr', "tmp/input", "tmp/files", "tmp/splits.txt",
"tmp/failures")
++        self.execute('hadoop', 'fs', '-mkdir', "tmp/input")
          self.comment("Starting bulk ingest example")
          self.comment("   Creating some test data")
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData',
0, 1000000, '/tmp/input/data')
-         self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable',
-                  INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
-         self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample',
-                  INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input',
'/tmp')
-         self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest',
-                  INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000)
 -        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData',
0, 1000000, '/tmp/input/data')
++        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData',
'--start-row', 0, '--count', 1000000, '--output', 'tmp/input/data')
+         self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable',
 -                 INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
++                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
 '-t', 'bulkTable')
+         self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample',
 -                 INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input',
'/tmp')
 -        self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
 -                 INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000)
++                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
 '-t', 'bulkTable', '--inputDir', 'tmp/input', '--workDir', 'tmp')
++        self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
++                     '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD,
 '-t', 'bulkTable', '--start-row', 0, '--count', 1000000)
          self.wait(self.runOn(self.masterHost(), [
--            'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines"
++            'hadoop', 'fs', '-rmr', "tmp/tableFile", "tmp/nines"
              ]))
          self.comment("Running TeraSortIngest for a million rows")
++        self.ashell('createtable sorted\nquit\n')
          # 10,000 times smaller than the real terasort
          ROWS = 1000*1000
          self.wait(self.runOn(self.masterHost(), [
              ACCUMULO_HOME+'/bin/tool.sh',
              examplesJar,
-             'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest',
+             'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
 -            ROWS,  
 -            10, 10,
 -            78, 78,
 -            'sorted',
 -            INSTANCE_NAME,
 -            ZOOKEEPERS,
 -            ROOT,
 -            ROOT_PASSWORD,
 -            4]))
 +            '--count', ROWS,  
 +            '-nk', 10, '-xk', 10,
 +            '-nk', 78, '-xk', 78,
 +            '-t', 'sorted',
 +            '-i', INSTANCE_NAME,
 +            '-z', ZOOKEEPERS,
 +            '-u', ROOT,
 +            '-p', ROOT_PASSWORD,
 +            '--splits', 4]))
          self.comment("Looking for '999' in all rows")
++        self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
          self.wait(self.runOn(self.masterHost(), [
              ACCUMULO_HOME+'/bin/tool.sh',
              examplesJar,
-             'org.apache.accumulo.simple.examples.mapreduce.RegexExample',
+             'org.apache.accumulo.examples.simple.mapreduce.RegexExample',
 -            INSTANCE_NAME,
 -            ZOOKEEPERS,
 -            ROOT,
 -            ROOT_PASSWORD,
 -            'sorted',
 -            '.*999.*',
 -            '.*',
 -            '.*',
 -            '.*',
 -            '/tmp/nines']))
 +            '-i', INSTANCE_NAME,
 +            '-z', ZOOKEEPERS,
 +            '-u', ROOT,
 +            '-p', ROOT_PASSWORD,
 +            '-t', 'sorted',
 +            '--rowRegex', '.*999.*',
-             '/tmp/nines']))
++            'tmp/nines']))
          self.comment("Generating hashes of each row into a new table")
++        self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
          self.wait(self.runOn(self.masterHost(), [
              ACCUMULO_HOME+'/bin/tool.sh',
              examplesJar,
-             'org.apache.accumulo.simple.examples.mapreduce.RowHash',
+             'org.apache.accumulo.examples.simple.mapreduce.RowHash',
 -            INSTANCE_NAME,
 -            ZOOKEEPERS,
 -            ROOT,
 -            ROOT_PASSWORD,
 -            'sorted',
 -            ':',
 +            '-i', INSTANCE_NAME,
 +            '-z', ZOOKEEPERS,
 +            '-u', ROOT,
 +            '-p', ROOT_PASSWORD,
 +            '-t', 'sorted',
 +            '--column', ':',
              'sortedHashed',
              ]))
          self.comment("Exporting the table to HDFS")
++        self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
          self.wait(self.runOn(self.masterHost(), [
              ACCUMULO_HOME+'/bin/tool.sh',
              examplesJar,
-             'org.apache.accumulo.simple.examples.mapreduce.TableToFile',
+             'org.apache.accumulo.examples.simple.mapreduce.TableToFile',
 -            INSTANCE_NAME,
 -            ZOOKEEPERS,
 -            ROOT,
 -            ROOT_PASSWORD,
 -            'sortedHashed',
 -            ',',
 -            '/tmp/tableFile'
 +            '-i', INSTANCE_NAME,
 +            '-z', ZOOKEEPERS,
 +            '-u', ROOT,
 +            '-p', ROOT_PASSWORD,
 +            '-t', 'sorted',
-             '--output', '/tmp/tableFile'
++            '--output', 'tmp/tableFile'
              ]))
          self.comment("Running WordCount using Accumulo aggregators")
          self.wait(self.runOn(self.masterHost(), [
--            'hadoop', 'fs', '-rmr', "/tmp/wc"
++            'hadoop', 'fs', '-rmr', "tmp/wc"
              ]))
          self.wait(self.runOn(self.masterHost(), [
--            'hadoop', 'fs', '-mkdir', "/tmp/wc"
++            'hadoop', 'fs', '-mkdir', "tmp/wc"
              ]))
          self.wait(self.runOn(self.masterHost(), [
--            'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "/tmp/wc/Accumulo.README"
++            'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "tmp/wc/Accumulo.README"
              ]))
 -        self.ashell('createtable wordCount -a count=org.apache.accumulo.core.iterators.aggregation.StringSummation\nquit\n')
 +        self.ashell('createtable wordCount\nsetiter -scan -majc -minc -p 10 -n sum -class
org.apache.accumulo.core.iterators.user.SummingCombiner\n\ncount\n\nSTRING\nquit\n')
          self.wait(self.runOn(self.masterHost(), [
              ACCUMULO_HOME+'/bin/tool.sh',
              examplesJar,
-             'org.apache.accumulo.simple.examples.mapreduce.WordCount',
+             'org.apache.accumulo.examples.simple.mapreduce.WordCount',
 -            INSTANCE_NAME,
 -            ZOOKEEPERS,
 -            '/tmp/wc',
 -            'wctable'
 +            '-i', INSTANCE_NAME,
 +            '-z', ZOOKEEPERS,
-             '--input', '/tmp/wc',
++            '-u', ROOT,
++            '-p', ROOT_PASSWORD,
++            '--input', 'tmp/wc',
 +            '-t', 'wctable'
              ]))
          self.comment("Inserting data with a batch writer")
-         self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter',
+         self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter',
 -                        INSTANCE_NAME,
 -                        ZOOKEEPERS,
 -                        ROOT,
 -                        ROOT_PASSWORD,
 -                        'helloBatch'])
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-t', 'helloBatch',
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD])
          self.comment("Reading data")
-         self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData',
+         self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData',
 -                         INSTANCE_NAME,
 -                         ZOOKEEPERS,
 -                         ROOT,
 -                         ROOT_PASSWORD,
 -                         'helloBatch'])
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-t', 'helloBatch',
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD])
          self.comment("Running isolated scans")
-         self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+         self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
 -                         INSTANCE_NAME,
 -                         ZOOKEEPERS,
 -                         ROOT,
 -                         ROOT_PASSWORD,
 -                         'itest1',
 -                         100000,
 -                         'true'])
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD,
 +                         '-t', 'itest1',
 +                         '--iterations', 100000,
 +                         '--isolated'])
          self.comment("Running scans without isolation")
-         self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+         self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
 -                         INSTANCE_NAME,
 -                         ZOOKEEPERS,
 -                         ROOT,
 -                         ROOT_PASSWORD,
 -                         'itest2',
 -                         100000,
 -                         'false'])
 -        self.comment("Inserting data using a map/reduce job")
 -        self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat',
 -                         INSTANCE_NAME,
 -                         ZOOKEEPERS,
 -                         ROOT,
 -                         ROOT_PASSWORD,
 -                        'helloOutputFormat'])
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD,
 +                         '-t', 'itest2',
 +                         '--iterations', 100000])
          self.comment("Using some example constraints")
          self.ashell('\n'.join([
              'createtable testConstraints',
@@@ -262,83 -275,87 +272,84 @@@
              'insert r1 cf1 cq1 ABC',
              'scan',
              'quit'
 -            ]))
 +            ]), 1)
          self.comment("Performing some row operations")
-         self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations', 
 -        self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
 -                           INSTANCE_NAME,
 -                           ZOOKEEPERS,
 -                           ROOT,
 -                           ROOT_PASSWORD])
++        self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations', 
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD ])
          self.comment("Using the batch writer")
-         self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter',
+         self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter',
 -                           INSTANCE_NAME, 
 -                           ZOOKEEPERS, 
 -                           ROOT, 
 -                           ROOT_PASSWORD, 
 -                           table,
 -                           min,
 -                           count,
 -                           valueSize,
 -                           memory,
 -                           latency,
 -                           numThreads,
 -                           visibility])
 +                        '-i', INSTANCE_NAME,
 +                        '-z', ZOOKEEPERS,
 +                        '-u', ROOT,
 +                        '-p', ROOT_PASSWORD,
 +                         '-t', table,
 +                         '--start', min,
 +                         '--num', count,
 +                         '--size', valueSize,
 +                         '--batchMemory', memory,
 +                         '--batchLatency', latency,
 +                         '--batchThreads', numThreads,
 +                         '--vis', visibility])
          self.comment("Reading and writing some data")
-         self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+         self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
                             '-i', INSTANCE_NAME, 
                             '-z', ZOOKEEPERS, 
                             '-u', ROOT, 
                             '-p', ROOT_PASSWORD, 
 -                           '-s', auths,
 -                           '-t', table,
 -                           '-e', 
 -                           '-r', 
 -                           '-dbg'])
 +                           '--auths', auths,
-                            '-t', table,
-                            '--createtable', 
++                           '--table', table,
 +                           '-c', 
 +                           '--debug'])
          self.comment("Deleting some data")
-         self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+         self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
                             '-i', INSTANCE_NAME, 
                             '-z', ZOOKEEPERS, 
                             '-u', ROOT, 
                             '-p', ROOT_PASSWORD, 
--                           '-s', auths,
--                           '-t', table,
++                           '-auths', auths,
++                           '--table', table,
                             '-d', 
 -                           '-dbg'])
 +                           '--debug'])
          self.comment("Writing some random data with the batch writer")
-         self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter',
+         self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter',
 -                           '-s',
 -                           5,
 -                           INSTANCE_NAME, 
 -                           ZOOKEEPERS, 
 -                           ROOT, 
 -                           ROOT_PASSWORD, 
 -                           table,
 -                           count, 
 -                           min, 
 -                           max, 
 -                           valueSize, 
 -                           memory, 
 -                           latency, 
 -                           numThreads, 
 -                           visibility])
 -        self.comment("Reading some random data with the batch scanner")
 +                           '-i', INSTANCE_NAME, 
 +                           '-z', ZOOKEEPERS, 
 +                           '-u', ROOT, 
 +                           '-p', ROOT_PASSWORD, 
 +                           '-t', table,
++                           '--seed','5',
 +                           '--num', count, 
 +                           '--min', min, 
 +                           '--max', max, 
 +                           '--size', valueSize, 
 +                           '--batchMemory', memory, 
 +                           '--batchLatency', latency, 
 +                           '--batchThreads', numThreads, 
 +                           '--vis', visibility])
 +        self.comment("Writing some random data with the batch writer")
-         self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner',
+         self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner',
 -                           '-s',
 -                           5,
 -                           INSTANCE_NAME, 
 -                           ZOOKEEPERS, 
 -                           ROOT, 
 -                           ROOT_PASSWORD, 
 -                           table,
 -                           count, 
 -                           min, 
 -                           max, 
 -                           valueSize, 
 -                           numThreads, 
 -                           auths]);
 +                           '-i', INSTANCE_NAME, 
 +                           '-z', ZOOKEEPERS, 
 +                           '-u', ROOT, 
 +                           '-p', ROOT_PASSWORD, 
 +                           '-t', table,
++                           '--seed','5',
 +                           '--num', count, 
 +                           '--min', min, 
 +                           '--max', max, 
 +                           '--size', valueSize, 
 +                           '--scanThreads', numThreads, 
 +                           '--auths', auths]);
          self.comment("Running an example table operation (Flush)")
-         self.runExample(['org.apache.accumulo.simple.examples.client.Flush',
+         self.runExample(['org.apache.accumulo.examples.simple.client.Flush',
 -                           INSTANCE_NAME,
 -                           ZOOKEEPERS,
 -                           ROOT,
 -                           ROOT_PASSWORD,
 -                           table])
 +                           '-i', INSTANCE_NAME, 
 +                           '-z', ZOOKEEPERS, 
 +                           '-u', ROOT, 
 +                           '-p', ROOT_PASSWORD, 
 +                           '-t', table])
          self.shutdown_accumulo();
  
  


Mime
View raw message