Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8069A105AB for ; Mon, 25 Nov 2013 20:39:02 +0000 (UTC) Received: (qmail 81930 invoked by uid 500); 25 Nov 2013 20:39:02 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 81900 invoked by uid 500); 25 Nov 2013 20:39:02 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 81893 invoked by uid 99); 25 Nov 2013 20:39:02 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 25 Nov 2013 20:39:02 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E908E910334; Mon, 25 Nov 2013 20:39:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ecn@apache.org To: commits@accumulo.apache.org Date: Mon, 25 Nov 2013 20:39:02 -0000 Message-Id: <1453961c45f049489f7d19510e0819fb@git.apache.org> In-Reply-To: <9085d6ad114e467fb5346e93baa9e300@git.apache.org> References: <9085d6ad114e467fb5346e93baa9e300@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Conflicts: examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java test/system/auto/simple/examples.py In addition to merging from 1.4 also fixed a lot of issues with examples functional test that were not issues in 1.4. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c175bd31 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c175bd31 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c175bd31 Branch: refs/heads/1.6.0-SNAPSHOT Commit: c175bd31cfb18a081e328ccf5b172f67fcea7491 Parents: 17ac692 1fe5f7f Author: Keith Turner Authored: Mon Nov 25 13:14:32 2013 -0500 Committer: Keith Turner Committed: Mon Nov 25 13:14:32 2013 -0500 ---------------------------------------------------------------------- .../simple/client/RandomBatchScanner.java | 21 ++-- .../examples/simple/mapreduce/TableToFile.java | 2 +- test/system/auto/simple/examples.py | 103 ++++++++++--------- 3 files changed, 73 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java ---------------------------------------------------------------------- diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java index 5194b67,0000000..2ae82b4 mode 100644,000000..100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java @@@ -1,225 -1,0 +1,234 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.examples.simple.client; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.Random; + +import org.apache.accumulo.core.cli.BatchScannerOpts; +import org.apache.accumulo.core.cli.ClientOnRequiredTable; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; + +import com.beust.jcommander.Parameter; + +/** + * Internal class used to verify validity of data read. + */ +class CountingVerifyingReceiver { + private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class); + + long count = 0; + int expectedValueSize = 0; + HashMap expectedRows; + + CountingVerifyingReceiver(HashMap expectedRows, int expectedValueSize) { + this.expectedRows = expectedRows; + this.expectedValueSize = expectedValueSize; + } + + public void receive(Key key, Value value) { + + String row = key.getRow().toString(); + long rowid = Integer.parseInt(row.split("_")[1]); + + byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize); + + if (!Arrays.equals(expectedValue, value.get())) { + log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get())); + } + + if (!expectedRows.containsKey(key.getRow())) { + log.error("Got unexpected key " + key); + } else { + expectedRows.put(key.getRow(), true); + } + + count++; + } +} + +/** + * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch for instructions. + */ +public class RandomBatchScanner { + private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class); + + /** + * Generate a number of ranges, each covering a single random row. + * + * @param num + * the number of ranges to generate + * @param min + * the minimum row that will be generated + * @param max + * the maximum row that will be generated + * @param r + * a random number generator + * @param ranges + * a set in which to store the generated ranges + * @param expectedRows + * a map in which to store the rows covered by the ranges (initially mapped to false) + */ + static void generateRandomQueries(int num, long min, long max, Random r, HashSet ranges, HashMap expectedRows) { + log.info(String.format("Generating %,d random queries...", num)); + while (ranges.size() < num) { + long rowid = (Math.abs(r.nextLong()) % (max - min)) + min; + + Text row1 = new Text(String.format("row_%010d", rowid)); + + Range range = new Range(new Text(row1)); + ranges.add(range); + expectedRows.put(row1, false); + } + + log.info("finished"); + } + + /** + * Prints a count of the number of rows mapped to false. + * + * @param expectedRows ++ * @return boolean indicating "were all the rows found?" + */ - private static void printRowsNotFound(HashMap expectedRows) { ++ private static boolean checkAllRowsFound(HashMap expectedRows) { + int count = 0; ++ boolean allFound = true; + for (Entry entry : expectedRows.entrySet()) + if (!entry.getValue()) + count++; + - if (count > 0) ++ if (count > 0) { + log.warn("Did not find " + count + " rows"); ++ allFound = false; ++ } ++ return allFound; + } + + /** + * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by + * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results. + * + * @param num + * the number of queries to generate + * @param min + * the min row to query + * @param max + * the max row to query + * @param evs + * the expected size of the values + * @param r + * a random number generator + * @param tsbr + * a batch scanner ++ * @return boolean indicating "did the queries go fine?" + */ - static void doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) { ++ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) { + + HashSet ranges = new HashSet(num); + HashMap expectedRows = new java.util.HashMap(); + + generateRandomQueries(num, min, max, r, ranges, expectedRows); + + tsbr.setRanges(ranges); + + CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs); + + long t1 = System.currentTimeMillis(); + + for (Entry entry : tsbr) { + receiver.receive(entry.getKey(), entry.getValue()); + } + + long t2 = System.currentTimeMillis(); + + log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0))); + log.info(String.format("num results : %,d%n", receiver.count)); + - printRowsNotFound(expectedRows); ++ return checkAllRowsFound(expectedRows); + } + + public static class Opts extends ClientOnRequiredTable { + @Parameter(names="--min", description="miniumum row that will be generated") + long min = 0; + @Parameter(names="--max", description="maximum ow that will be generated") + long max = 0; + @Parameter(names="--num", required=true, description="number of ranges to generate") + int num = 0; + @Parameter(names="--size", required=true, description="size of the value to write") + int size = 0; + @Parameter(names="--seed", description="seed for pseudo-random number generator") + Long seed = null; + } + + /** + * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for + * a repeated query which has cached metadata and connections already established. + * + * @param args + * @throws AccumuloException + * @throws AccumuloSecurityException + * @throws TableNotFoundException + */ + public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + Opts opts = new Opts(); + BatchScannerOpts bsOpts = new BatchScannerOpts(); + opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts); + + Connector connector = opts.getConnector(); + BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads); + batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS); + + Random r; + if (opts.seed == null) + r = new Random(); + else + r = new Random(opts.seed); + + // do one cold - doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); ++ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); + + System.gc(); + System.gc(); + System.gc(); + + if (opts.seed == null) + r = new Random(); + else + r = new Random(opts.seed); + + // do one hot (connections already established, metadata table cached) - doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); ++ status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader); + + batchReader.close(); ++ if (!status) { ++ System.exit(1); ++ } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java ---------------------------------------------------------------------- diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java index 4951852,0000000..d8eedef mode 100644,000000..100644 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java @@@ -1,130 -1,0 +1,130 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.examples.simple.mapreduce; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; + +import org.apache.accumulo.core.cli.ClientOnRequiredTable; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.util.format.DefaultFormatter; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import com.beust.jcommander.Parameter; + +/** + * Takes a table and outputs the specified column to a set of part files on hdfs accumulo accumulo.examples.mapreduce.TableToFile + * + */ +public class TableToFile extends Configured implements Tool { + + static class Opts extends ClientOnRequiredTable { + @Parameter(names = "--output", description = "output directory", required = true) + String output; + @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form") - String columns; ++ String columns = ""; + } + + /** + * The Mapper class that given a row number, will generate the appropriate output line. + */ + public static class TTFMapper extends Mapper { + @Override + public void map(Key row, Value data, Context context) throws IOException, InterruptedException { + final Key r = row; + final Value v = data; + Map.Entry entry = new Map.Entry() { + @Override + public Key getKey() { + return r; + } + + @Override + public Value getValue() { + return v; + } + + @Override + public Value setValue(Value value) { + return null; + } + }; + context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false))); + context.setStatus("Outputed Value"); + } + } + + @Override + public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException { + Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); + job.setJarByClass(this.getClass()); + Opts opts = new Opts(); + opts.parseArgs(getClass().getName(), args); + + job.setInputFormatClass(AccumuloInputFormat.class); + opts.setAccumuloConfigs(job); + + HashSet> columnsToFetch = new HashSet>(); + for (String col : opts.columns.split(",")) { + int idx = col.indexOf(":"); + Text cf = new Text(idx < 0 ? col : col.substring(0, idx)); + Text cq = idx < 0 ? null : new Text(col.substring(idx + 1)); + if (cf.getLength() > 0) + columnsToFetch.add(new Pair(cf, cq)); + } + if (!columnsToFetch.isEmpty()) + AccumuloInputFormat.fetchColumns(job, columnsToFetch); + + job.setMapperClass(TTFMapper.class); + job.setMapOutputKeyClass(NullWritable.class); + job.setMapOutputValueClass(Text.class); + + job.setNumReduceTasks(0); + + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, new Path(opts.output)); + + job.waitForCompletion(true); + return job.isSuccessful() ? 0 : 1; + } + + /** + * + * @param args + * instanceName zookeepers username password table columns outputpath + * @throws Exception + */ + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args); + if (res != 0) + System.exit(res); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/test/system/auto/simple/examples.py ---------------------------------------------------------------------- diff --cc test/system/auto/simple/examples.py index 069148d,b22411a..3ba425e --- a/test/system/auto/simple/examples.py +++ b/test/system/auto/simple/examples.py @@@ -41,11 -41,15 +41,11 @@@ class Examples(TestUtilsMixin, unittest order = 21 def runExample(self, cmd): - return self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)) + self.assert_(self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)), "example exited with error status.") - def ignoreExample(self, cmd): - self.comment(" Ignoring results of command.") - self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)) - - def ashell(self, input): + def ashell(self, input, expected = 0): out, err, code = self.shell(self.masterHost(), input + '\n') - self.assert_(code == 0) + self.assert_(code == expected) return out def comment(self, description): @@@ -58,17 -68,9 +64,16 @@@ self.wait(self.runOn('localhost', cmd)) def runTest(self): + examplesJar = glob.glob(ACCUMULO_HOME+'/lib/accumulo-examples-simple*.jar')[0] - + self.comment("Testing MaxMutation constraint") + self.ashell('createtable test_ingest\n' + 'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n') + handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD]) + out, err = handle.communicate() + self.failIf(handle.returncode==0) + self.failUnless(err.find("MutationsRejectedException: # constraint violations : 1") >= 0, "Was able to insert a mutation larger than max size") + self.ashell('createtable %s\nsetauths -u %s -s A,B\nquit\n' %(table, ROOT)) - examplesJar = glob.glob(ACCUMULO_HOME+'/lib/examples-simple*.jar')[0] - self.comment("Testing dirlist example (a little)") self.comment(" ingesting accumulo source") self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.dirlist.Ingest', @@@ -103,156 -105,167 +108,161 @@@ self.comment("Testing bloom filters are fast for missing data") self.ashell('createtable bloom_test\nconfig -t bloom_test -s table.bloom.enabled=true\n') - self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '-s', '7', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test', - '1000000', '0', '1000000000', '50', '2000000', '60000', '3', 'A') + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '--seed', '7', + '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test', + '--num', '1000000', '--min', '0', '--max', '1000000000', '--size', '50', '--batchMemory', '2M', '--batchLatency', '60s', + '--batchThreads', '3') self.ashell('flush -t bloom_test -w\n') now = time.time() - self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '7', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test', - 500, 0, 1000000000, 50, 20, 'A') + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '7', + '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test', + '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50', '--scanThreads', 4) diff = time.time() - now now = time.time() - self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8', - self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test', - 500, 0, 1000000000, 50, 20, 'A') ++ self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8', + '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test', + '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50', '--scanThreads', 4) diff2 = time.time() - now self.assert_(diff2 < diff) self.comment("Creating a sharded index of the accumulo java files") self.ashell('createtable shard\ncreatetable doc2term\nquit\n') self.execute('/bin/sh', '-c', - 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.simple.examples.shard.Index -i %s -z %s -t shard -u %s -p %s --partitions 30' % - 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index %s %s shard %s %s 30' % ++ 'find server -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index -i %s -z %s -t shard -u %s -p %s --partitions 30' % (INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD)) - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Query', + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Query', - INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD, + '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '-u', ROOT, '-p', ROOT_PASSWORD, 'foo', 'bar') self.comment("Creating a word index of the sharded files") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Reverse', - '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD) + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Reverse', - INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD) ++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD) self.comment("Making 1000 conjunctive queries of 5 random words") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.ContinuousQuery', - '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--term', 5, '--count', 1000) - - self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures") - self.execute('hadoop', 'fs', '-mkdir', "/tmp/input") + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.ContinuousQuery', - INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD, 5, 1000) - - self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures") - self.execute('hadoop', 'fs', '-mkdir', "/tmp/input") ++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--terms', 5, '--count', 1000) ++ self.executeIgnoreFail('hadoop', 'fs', '-rmr', "tmp/input", "tmp/files", "tmp/splits.txt", "tmp/failures") ++ self.execute('hadoop', 'fs', '-mkdir', "tmp/input") self.comment("Starting bulk ingest example") self.comment(" Creating some test data") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data') - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable') - self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp') - self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000) - self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data') ++ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', '--start-row', 0, '--count', 1000000, '--output', 'tmp/input/data') + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable') ++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable') + self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp') - self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest', - INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000) ++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable', '--inputDir', 'tmp/input', '--workDir', 'tmp') ++ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest', ++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable', '--start-row', 0, '--count', 1000000) self.wait(self.runOn(self.masterHost(), [ -- 'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines" ++ 'hadoop', 'fs', '-rmr', "tmp/tableFile", "tmp/nines" ])) self.comment("Running TeraSortIngest for a million rows") ++ self.ashell('createtable sorted\nquit\n') # 10,000 times smaller than the real terasort ROWS = 1000*1000 self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest', + 'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest', - ROWS, - 10, 10, - 78, 78, - 'sorted', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 4])) + '--count', ROWS, + '-nk', 10, '-xk', 10, + '-nk', 78, '-xk', 78, + '-t', 'sorted', + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '--splits', 4])) self.comment("Looking for '999' in all rows") ++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n') self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.RegexExample', + 'org.apache.accumulo.examples.simple.mapreduce.RegexExample', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'sorted', - '.*999.*', - '.*', - '.*', - '.*', - '/tmp/nines'])) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', 'sorted', + '--rowRegex', '.*999.*', - '/tmp/nines'])) ++ 'tmp/nines'])) self.comment("Generating hashes of each row into a new table") ++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n') self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.RowHash', + 'org.apache.accumulo.examples.simple.mapreduce.RowHash', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'sorted', - ':', + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', 'sorted', + '--column', ':', 'sortedHashed', ])) self.comment("Exporting the table to HDFS") ++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n') self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.TableToFile', + 'org.apache.accumulo.examples.simple.mapreduce.TableToFile', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'sortedHashed', - ',', - '/tmp/tableFile' + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', 'sorted', - '--output', '/tmp/tableFile' ++ '--output', 'tmp/tableFile' ])) self.comment("Running WordCount using Accumulo aggregators") self.wait(self.runOn(self.masterHost(), [ -- 'hadoop', 'fs', '-rmr', "/tmp/wc" ++ 'hadoop', 'fs', '-rmr', "tmp/wc" ])) self.wait(self.runOn(self.masterHost(), [ -- 'hadoop', 'fs', '-mkdir', "/tmp/wc" ++ 'hadoop', 'fs', '-mkdir', "tmp/wc" ])) self.wait(self.runOn(self.masterHost(), [ -- 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "/tmp/wc/Accumulo.README" ++ 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "tmp/wc/Accumulo.README" ])) - self.ashell('createtable wordCount -a count=org.apache.accumulo.core.iterators.aggregation.StringSummation\nquit\n') + self.ashell('createtable wordCount\nsetiter -scan -majc -minc -p 10 -n sum -class org.apache.accumulo.core.iterators.user.SummingCombiner\n\ncount\n\nSTRING\nquit\n') self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.WordCount', + 'org.apache.accumulo.examples.simple.mapreduce.WordCount', - INSTANCE_NAME, - ZOOKEEPERS, - '/tmp/wc', - 'wctable' + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, - '--input', '/tmp/wc', ++ '-u', ROOT, ++ '-p', ROOT_PASSWORD, ++ '--input', 'tmp/wc', + '-t', 'wctable' ])) self.comment("Inserting data with a batch writer") - self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'helloBatch']) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-t', 'helloBatch', + '-u', ROOT, + '-p', ROOT_PASSWORD]) self.comment("Reading data") - self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData', + self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'helloBatch']) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-t', 'helloBatch', + '-u', ROOT, + '-p', ROOT_PASSWORD]) self.comment("Running isolated scans") - self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest', + self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'itest1', - 100000, - 'true']) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', 'itest1', + '--iterations', 100000, + '--isolated']) self.comment("Running scans without isolation") - self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest', + self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'itest2', - 100000, - 'false']) - self.comment("Inserting data using a map/reduce job") - self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - 'helloOutputFormat']) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', 'itest2', + '--iterations', 100000]) self.comment("Using some example constraints") self.ashell('\n'.join([ 'createtable testConstraints', @@@ -262,83 -275,87 +272,84 @@@ 'insert r1 cf1 cq1 ABC', 'scan', 'quit' - ])) + ]), 1) self.comment("Performing some row operations") - self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations', - self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD]) ++ self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations', + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD ]) self.comment("Using the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - table, - min, - count, - valueSize, - memory, - latency, - numThreads, - visibility]) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', table, + '--start', min, + '--num', count, + '--size', valueSize, + '--batchMemory', memory, + '--batchLatency', latency, + '--batchThreads', numThreads, + '--vis', visibility]) self.comment("Reading and writing some data") - self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample', + self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample', '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, - '-s', auths, - '-t', table, - '-e', - '-r', - '-dbg']) + '--auths', auths, - '-t', table, - '--createtable', ++ '--table', table, + '-c', + '--debug']) self.comment("Deleting some data") - self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample', + self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample', '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, -- '-s', auths, -- '-t', table, ++ '-auths', auths, ++ '--table', table, '-d', - '-dbg']) + '--debug']) self.comment("Writing some random data with the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter', - '-s', - 5, - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - table, - count, - min, - max, - valueSize, - memory, - latency, - numThreads, - visibility]) - self.comment("Reading some random data with the batch scanner") + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', table, ++ '--seed','5', + '--num', count, + '--min', min, + '--max', max, + '--size', valueSize, + '--batchMemory', memory, + '--batchLatency', latency, + '--batchThreads', numThreads, + '--vis', visibility]) + self.comment("Writing some random data with the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner', + self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner', - '-s', - 5, - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - table, - count, - min, - max, - valueSize, - numThreads, - auths]); + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', table, ++ '--seed','5', + '--num', count, + '--min', min, + '--max', max, + '--size', valueSize, + '--scanThreads', numThreads, + '--auths', auths]); self.comment("Running an example table operation (Flush)") - self.runExample(['org.apache.accumulo.simple.examples.client.Flush', + self.runExample(['org.apache.accumulo.examples.simple.client.Flush', - INSTANCE_NAME, - ZOOKEEPERS, - ROOT, - ROOT_PASSWORD, - table]) + '-i', INSTANCE_NAME, + '-z', ZOOKEEPERS, + '-u', ROOT, + '-p', ROOT_PASSWORD, + '-t', table]) self.shutdown_accumulo();