Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E97DF10DE4 for ; Mon, 25 Nov 2013 18:19:07 +0000 (UTC) Received: (qmail 34010 invoked by uid 500); 25 Nov 2013 18:18:11 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 33969 invoked by uid 500); 25 Nov 2013 18:18:08 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 33946 invoked by uid 99); 25 Nov 2013 18:18:05 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 25 Nov 2013 18:18:05 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id CD6FA9070A6; Mon, 25 Nov 2013 18:18:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kturner@apache.org To: commits@accumulo.apache.org Date: Mon, 25 Nov 2013 18:18:04 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: ACCUMULO-1878 handle error conditions in example tests. Updated Branches: refs/heads/1.5.1-SNAPSHOT 17ac69203 -> c175bd31c ACCUMULO-1878 handle error conditions in example tests. * ensure simple examples used in integration tests properly set non-zero exit on errors * check return code of executed commands. * fix typo in bloom filter speed comparison * fix error in arg order on helloworld examples * Makes sure paired examples for RandomBatch use the same seed. Signed-off-by: Keith Turner Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1fe5f7f5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1fe5f7f5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1fe5f7f5 Branch: refs/heads/1.5.1-SNAPSHOT Commit: 1fe5f7f528d39746c0e5cffb8710b4db21c3a5b2 Parents: d0ceebb Author: Sean Busbey Authored: Thu Nov 7 09:07:22 2013 -0600 Committer: Keith Turner Committed: Mon Nov 25 09:36:39 2013 -0500 ---------------------------------------------------------------------- .../simple/client/RandomBatchScanner.java | 24 ++++-- test/system/auto/simple/examples.py | 88 ++++++++++++-------- 2 files changed, 68 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/1fe5f7f5/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java ---------------------------------------------------------------------- diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java index 7801104..604e14e 100644 --- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java +++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java @@ -112,15 +112,20 @@ public class RandomBatchScanner { * Prints a count of the number of rows mapped to false. * * @param expectedRows + * @return boolean indicating "were all the rows found?" */ - private static void printRowsNotFound(HashMap expectedRows) { + private static boolean checkAllRowsFound(HashMap expectedRows) { int count = 0; + boolean allFound = true; for (Entry entry : expectedRows.entrySet()) if (!entry.getValue()) count++; - if (count > 0) + if (count > 0) { log.warn("Did not find " + count + " rows"); + allFound = false; + } + return allFound; } /** @@ -139,8 +144,9 @@ public class RandomBatchScanner { * a random number generator * @param tsbr * a batch scanner + * @return boolean indicating "did the queries go fine?" */ - static void doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) { + static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) { HashSet ranges = new HashSet(num); HashMap expectedRows = new java.util.HashMap(); @@ -162,7 +168,7 @@ public class RandomBatchScanner { log.info(String.format("%6.2f lookups/sec %6.2f secs\n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0))); log.info(String.format("num results : %,d\n", receiver.count)); - printRowsNotFound(expectedRows); + return checkAllRowsFound(expectedRows); } /** @@ -175,6 +181,7 @@ public class RandomBatchScanner { * @throws TableNotFoundException */ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + boolean status = true; String seed = null; int index = 0; @@ -190,7 +197,7 @@ public class RandomBatchScanner { if (index != 11) { System.out .println("Usage : RandomBatchScanner [-s ] "); - return; + System.exit(1); } String instanceName = processedArgs[0]; @@ -220,7 +227,7 @@ public class RandomBatchScanner { r = new Random(Long.parseLong(seed)); // do one cold - doRandomQueries(num, min, max, expectedValueSize, r, tsbr); + status = doRandomQueries(num, min, max, expectedValueSize, r, tsbr); System.gc(); System.gc(); @@ -232,8 +239,11 @@ public class RandomBatchScanner { r = new Random(Long.parseLong(seed)); // do one hot (connections already established, metadata table cached) - doRandomQueries(num, min, max, expectedValueSize, r, tsbr); + status = status && doRandomQueries(num, min, max, expectedValueSize, r, tsbr); tsbr.close(); + if (!status) { + System.exit(1); + } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/1fe5f7f5/test/system/auto/simple/examples.py ---------------------------------------------------------------------- diff --git a/test/system/auto/simple/examples.py b/test/system/auto/simple/examples.py index 5fc9b4b..b22411a 100755 --- a/test/system/auto/simple/examples.py +++ b/test/system/auto/simple/examples.py @@ -41,6 +41,10 @@ class Examples(TestUtilsMixin, unittest.TestCase): order = 21 def runExample(self, cmd): + self.assert_(self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)), "example exited with error status.") + + def ignoreExample(self, cmd): + self.comment(" Ignoring results of command.") self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)) def ashell(self, input): @@ -55,6 +59,12 @@ class Examples(TestUtilsMixin, unittest.TestCase): log.info(LINE) def execute(self, *cmd): + self.assert_(self.wait(self.runOn('localhost', cmd)), "command exited with error status.") + + def executeExpectFail(self, *cmd): + self.assert_(not self.wait(self.runOn('localhost', cmd)), "command did not exit with error status and we expected it to.") + + def executeIgnoreFail(self, *cmd): self.wait(self.runOn('localhost', cmd)) def runTest(self): @@ -97,7 +107,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.ashell('createtable bloom_test\nconfig -t bloom_test -s table.bloom.enabled=true\n') self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '-s', '7', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test', - '1000000', '0', '1000000000', '50', '2000000', '60000', '3' 'A') + '1000000', '0', '1000000000', '50', '2000000', '60000', '3', 'A') self.ashell('flush -t bloom_test -w\n') now = time.time() self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '7', @@ -105,7 +115,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): 500, 0, 1000000000, 50, 20, 'A') diff = time.time() - now now = time.time() - self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8', + self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test', 500, 0, 1000000000, 50, 20, 'A') diff2 = time.time() - now @@ -114,28 +124,28 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.comment("Creating a sharded index of the accumulo java files") self.ashell('createtable shard\ncreatetable doc2term\nquit\n') self.execute('/bin/sh', '-c', - 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.simple.examples.shard.Index %s %s shard %s %s 30' % + 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index %s %s shard %s %s 30' % (INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD)) - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Query', + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Query', INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD, 'foo', 'bar') self.comment("Creating a word index of the sharded files") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Reverse', + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Reverse', INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD) self.comment("Making 1000 conjunctive queries of 5 random words") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.ContinuousQuery', + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.ContinuousQuery', INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD, 5, 1000) - self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures") + self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures") self.execute('hadoop', 'fs', '-mkdir', "/tmp/input") self.comment("Starting bulk ingest example") self.comment(" Creating some test data") - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data') - self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable', + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data') + self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable') - self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample', + self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp') - self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest', + self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000) self.wait(self.runOn(self.masterHost(), [ 'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines" @@ -146,7 +156,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest', + 'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest', ROWS, 10, 10, 78, 78, @@ -160,7 +170,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.RegexExample', + 'org.apache.accumulo.examples.simple.mapreduce.RegexExample', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -175,7 +185,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.RowHash', + 'org.apache.accumulo.examples.simple.mapreduce.RowHash', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -188,7 +198,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.TableToFile', + 'org.apache.accumulo.examples.simple.mapreduce.TableToFile', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -211,28 +221,28 @@ class Examples(TestUtilsMixin, unittest.TestCase): self.wait(self.runOn(self.masterHost(), [ ACCUMULO_HOME+'/bin/tool.sh', examplesJar, - 'org.apache.accumulo.simple.examples.mapreduce.WordCount', + 'org.apache.accumulo.examples.simple.mapreduce.WordCount', INSTANCE_NAME, ZOOKEEPERS, '/tmp/wc', 'wctable' ])) self.comment("Inserting data with a batch writer") - self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter', INSTANCE_NAME, ZOOKEEPERS, - 'helloBatch', ROOT, - ROOT_PASSWORD]) + ROOT_PASSWORD, + 'helloBatch']) self.comment("Reading data") - self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData', + self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData', INSTANCE_NAME, ZOOKEEPERS, - 'helloBatch', ROOT, - ROOT_PASSWORD]) + ROOT_PASSWORD, + 'helloBatch']) self.comment("Running isolated scans") - self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest', + self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -241,7 +251,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): 100000, 'true']) self.comment("Running scans without isolation") - self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest', + self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -250,30 +260,30 @@ class Examples(TestUtilsMixin, unittest.TestCase): 100000, 'false']) self.comment("Inserting data using a map/reduce job") - self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithOutputFormat', + self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat', INSTANCE_NAME, ZOOKEEPERS, - 'helloOutputFormat', ROOT, - ROOT_PASSWORD]) + ROOT_PASSWORD, + 'helloOutputFormat']) self.comment("Using some example constraints") self.ashell('\n'.join([ 'createtable testConstraints', - 'config -t testConstraints -s table.constraint.1=org.apache.accumulo.simple.examples.constraints.NumericValueConstraint', - 'config -t testConstraints -s table.constraint.2=org.apache.accumulo.simple.examples.constraints.AlphaNumKeyConstraint', + 'config -t testConstraints -s table.constraint.1=org.apache.accumulo.examples.simple.constraints.NumericValueConstraint', + 'config -t testConstraints -s table.constraint.2=org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint', 'insert r1 cf1 cq1 1111', 'insert r1 cf1 cq1 ABC', 'scan', 'quit' ])) self.comment("Performing some row operations") - self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations', + self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations', INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD]) self.comment("Using the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter', INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -287,7 +297,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): numThreads, visibility]) self.comment("Reading and writing some data") - self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample', + self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample', '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, @@ -298,7 +308,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): '-r', '-dbg']) self.comment("Deleting some data") - self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample', + self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample', '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, @@ -308,7 +318,9 @@ class Examples(TestUtilsMixin, unittest.TestCase): '-d', '-dbg']) self.comment("Writing some random data with the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter', + self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter', + '-s', + 5, INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -322,8 +334,10 @@ class Examples(TestUtilsMixin, unittest.TestCase): latency, numThreads, visibility]) - self.comment("Writing some random data with the batch writer") - self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner', + self.comment("Reading some random data with the batch scanner") + self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner', + '-s', + 5, INSTANCE_NAME, ZOOKEEPERS, ROOT, @@ -336,7 +350,7 @@ class Examples(TestUtilsMixin, unittest.TestCase): numThreads, auths]); self.comment("Running an example table operation (Flush)") - self.runExample(['org.apache.accumulo.simple.examples.client.Flush', + self.runExample(['org.apache.accumulo.examples.simple.client.Flush', INSTANCE_NAME, ZOOKEEPERS, ROOT,