Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 73E3010141 for ; Tue, 2 Dec 2014 20:36:51 +0000 (UTC) Received: (qmail 95842 invoked by uid 500); 2 Dec 2014 20:36:51 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 95707 invoked by uid 500); 2 Dec 2014 20:36:51 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 94407 invoked by uid 99); 2 Dec 2014 20:36:50 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Dec 2014 20:36:50 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 4BC4A9BBFBE; Tue, 2 Dec 2014 20:36:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Tue, 02 Dec 2014 20:37:13 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [25/50] [abbrv] accumulo git commit: ACCUMULO-3167 Get the non-mapreduce ExamplesIT running against standalone ACCUMULO-3167 Get the non-mapreduce ExamplesIT running against standalone Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76917802 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76917802 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76917802 Branch: refs/heads/metrics2 Commit: 769178020b6d2284dbd7013c48a4a008a0e2dd2f Parents: 0105730 Author: Josh Elser Authored: Mon Nov 24 01:25:33 2014 -0500 Committer: Josh Elser Committed: Mon Nov 24 18:08:52 2014 -0500 ---------------------------------------------------------------------- .../apache/accumulo/cluster/ClusterControl.java | 6 + .../standalone/StandaloneClusterControl.java | 7 +- .../impl/MiniAccumuloClusterControl.java | 35 +++ .../accumulo/harness/AccumuloClusterIT.java | 6 + .../accumulo/test/functional/ExamplesIT.java | 222 +++++++++++-------- 5 files changed, 186 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/76917802/minicluster/src/main/java/org/apache/accumulo/cluster/ClusterControl.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/ClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/ClusterControl.java index 0fcceb1..be4a8fd 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/ClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/ClusterControl.java @@ -17,6 +17,7 @@ package org.apache.accumulo.cluster; import java.io.IOException; +import java.util.Map.Entry; import org.apache.accumulo.minicluster.ServerType; @@ -31,6 +32,11 @@ public interface ClusterControl { int exec(Class clz, String[] args) throws IOException; /** + * Execute the given class against the cluster with the provided arguments and waits for completion. Returns the exit code of the process with the stdout. + */ + Entry execWithStdout(Class clz, String[] args) throws IOException; + + /** * Issue an orderly shutdown of the cluster */ void adminStopAll() throws IOException; http://git-wip-us.apache.org/repos/asf/accumulo/blob/76917802/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java index 9f93161..29cc1e8 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java @@ -86,6 +86,11 @@ public class StandaloneClusterControl implements ClusterControl { @Override public int exec(Class clz, String[] args) throws IOException { + return execWithStdout(clz, args).getKey(); + } + + @Override + public Entry execWithStdout(Class clz, String[] args) throws IOException { File confDir = getConfDir(); String master = getHosts(new File(confDir, "masters")).get(0); String[] cmd = new String[2 + args.length]; @@ -93,7 +98,7 @@ public class StandaloneClusterControl implements ClusterControl { cmd[1] = clz.getName(); System.arraycopy(args, 0, cmd, 2, args.length); log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), master); - return exec(master, cmd).getKey(); + return exec(master, cmd); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/76917802/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterControl.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterControl.java index ed90cbf..839285a 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterControl.java @@ -16,10 +16,13 @@ */ package org.apache.accumulo.minicluster.impl; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map.Entry; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -28,6 +31,7 @@ import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.master.Master; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.tracer.TraceServer; @@ -36,6 +40,7 @@ import org.apache.log4j.Logger; import org.apache.zookeeper.server.ZooKeeperServerMain; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; /** * @@ -76,6 +81,36 @@ public class MiniAccumuloClusterControl implements ClusterControl { } @Override + public Entry execWithStdout(Class clz, String[] args) throws IOException { + Process p = cluster.exec(clz, args); + int exitCode; + try { + exitCode = p.waitFor(); + } catch (InterruptedException e) { + log.warn("Interrupted waiting for process to exit", e); + Thread.currentThread().interrupt(); + throw new IOException(e); + } + for (LogWriter writer : cluster.getLogWriters()) { + writer.flush(); + } + return Maps.immutableEntry(exitCode, + readAll(new FileInputStream(cluster.getConfig().getLogDir() + "/" + clz.getSimpleName() + "_" + p.hashCode() + ".out"))); + } + + private String readAll(InputStream is) throws IOException { + byte[] buffer = new byte[4096]; + StringBuffer result = new StringBuffer(); + while (true) { + int n = is.read(buffer); + if (n <= 0) + break; + result.append(new String(buffer, 0, n)); + } + return result.toString(); + } + + @Override public void adminStopAll() throws IOException { Process p = cluster.exec(Admin.class, "stopAll"); try { http://git-wip-us.apache.org/repos/asf/accumulo/blob/76917802/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java index 621fb13..38b615f 100644 --- a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java +++ b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import org.apache.accumulo.cluster.AccumuloCluster; +import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.admin.SecurityOperations; @@ -145,6 +146,11 @@ public abstract class AccumuloClusterIT extends AccumuloIT implements MiniCluste return cluster; } + public static ClusterControl getClusterControl() { + Preconditions.checkState(initialized); + return cluster.getClusterControl(); + } + public static ClusterType getClusterType() { Preconditions.checkState(initialized); return type; http://git-wip-us.apache.org/repos/asf/accumulo/blob/76917802/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java index 210e057..047e69d 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java @@ -16,6 +16,7 @@ */ package org.apache.accumulo.test.functional; +import static com.google.common.base.Charsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -37,6 +38,7 @@ import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; @@ -44,7 +46,6 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.AgeOffFilter; import org.apache.accumulo.core.iterators.user.SummingCombiner; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.examples.simple.client.Flush; import org.apache.accumulo.examples.simple.client.RandomBatchScanner; @@ -73,7 +74,9 @@ import org.apache.accumulo.examples.simple.shard.ContinuousQuery; import org.apache.accumulo.examples.simple.shard.Index; import org.apache.accumulo.examples.simple.shard.Query; import org.apache.accumulo.examples.simple.shard.Reverse; +import org.apache.accumulo.harness.AccumuloClusterIT; import org.apache.accumulo.minicluster.MemoryUnit; +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl; import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.accumulo.start.Main; @@ -84,10 +87,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; -public class ExamplesIT extends ConfigurableMacIT { +public class ExamplesIT extends AccumuloClusterIT { private static final Logger log = Logger.getLogger(ExamplesIT.class); private static final BatchWriterOpts bwOpts = new BatchWriterOpts(); private static final BatchWriterConfig bwc = new BatchWriterConfig(); @@ -97,15 +103,16 @@ public class ExamplesIT extends ConfigurableMacIT { Connector c; String instance; String keepers; - String user = "root"; + String user; String passwd; BatchWriter bw; IteratorSetting is; String dir; FileSystem fs; + Authorizations origAuths; @Override - public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopConf) { + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopConf) { // 128MB * 3 cfg.setDefaultMemory(cfg.getDefaultMemory() * 3, MemoryUnit.BYTE); } @@ -113,15 +120,25 @@ public class ExamplesIT extends ConfigurableMacIT { @Before public void getClusterInfo() throws Exception { c = getConnector(); - passwd = AbstractMacIT.ROOT_PASSWORD; - fs = FileSystem.get(CachedConfiguration.getInstance()); + user = getPrincipal(); + Assume.assumeTrue(getToken() instanceof PasswordToken); + passwd = new String(((PasswordToken) getToken()).getPassword(), UTF_8); + fs = getCluster().getFileSystem(); instance = c.getInstance().getInstanceName(); keepers = c.getInstance().getZooKeepers(); - dir = cluster.getConfig().getDir().getAbsolutePath(); + dir = getUsableDir(); + origAuths = c.securityOperations().getUserAuthorizations(user); c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(","))); } + @After + public void resetAuths() throws Exception { + if (null != origAuths) { + getConnector().securityOperations().changeUserAuthorizations(getPrincipal(), origAuths); + } + } + @Override public int defaultTimeoutSeconds() { return 6 * 60; @@ -129,32 +146,38 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testTrace() throws Exception { - Process trace = cluster.exec(TraceServer.class); - while (!c.tableOperations().exists("trace")) - UtilWaitThread.sleep(500); - Process p = goodExec(TracingExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-C", "-D", "-c"); - for (LogWriter writer : cluster.getLogWriters()) { - writer.flush(); + Process trace = null; + if (ClusterType.MINI == getClusterType()) { + MiniAccumuloClusterImpl impl = (MiniAccumuloClusterImpl) cluster; + trace = impl.exec(TraceServer.class); + while (!c.tableOperations().exists("trace")) + UtilWaitThread.sleep(500); } - String result = FunctionalTestUtils.readAll(cluster, TracingExample.class, p); + Entry pair = cluster.getClusterControl().execWithStdout(TracingExample.class, + new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-C", "-D", "-c"}); + Assert.assertEquals("Expected return code of zero. STDOUT=" + pair.getValue(), 0, pair.getKey().intValue()); + String result = pair.getValue(); Pattern pattern = Pattern.compile("TraceID: ([0-9a-f]+)"); Matcher matcher = pattern.matcher(result); int count = 0; while (matcher.find()) { - p = goodExec(TraceDumpExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--traceid", matcher.group(1)); + pair = cluster.getClusterControl().execWithStdout(TraceDumpExample.class, + new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--traceid", matcher.group(1)}); count++; } assertTrue(count > 0); - result = FunctionalTestUtils.readAll(cluster, TraceDumpExample.class, p); - assertTrue(result.contains("myHost@myApp")); - trace.destroy(); + assertTrue(pair.getValue().contains("myHost@myApp")); + if (ClusterType.MINI == getClusterType() && null != trace) { + trace.destroy(); + } } @Test public void testClasspath() throws Exception { - Process p = cluster.exec(Main.class, Collections.singletonList(MapReduceIT.hadoopTmpDirArg), "classpath"); - assertEquals(0, p.waitFor()); - String result = FunctionalTestUtils.readAll(cluster, Main.class, p); + // Process p = cluster.exec(Main.class, Collections.singletonList(MapReduceIT.hadoopTmpDirArg), "classpath"); + Entry entry = getCluster().getClusterControl().execWithStdout(Main.class, new String[] {"classpath"}); + assertEquals(0, entry.getKey().intValue()); + String result = entry.getValue(); int level1 = result.indexOf("Level 1"); int level2 = result.indexOf("Level 2"); int level3 = result.indexOf("Level 3"); @@ -170,37 +193,50 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testDirList() throws Exception { - goodExec(Ingest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable", "dirTable", "--indexTable", "indexTable", "--dataTable", - "dataTable", "--vis", visibility, "--chunkSize", 10000 + "", cluster.getConfig().getDir().getAbsolutePath()); - Process p = goodExec(QueryUtil.class, "-i", instance, "-z", keepers, "-p", passwd, "-u", user, "-t", "indexTable", "--auths", auths, "--search", "--path", - "accumulo-site.xml"); - for (LogWriter writer : cluster.getLogWriters()) { - writer.flush(); + String[] names = getUniqueNames(3); + String dirTable = names[0], indexTable = names[1], dataTable = names[2]; + Entry entry = getClusterControl().execWithStdout( + Ingest.class, + new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable", dirTable, "--indexTable", indexTable, "--dataTable", dataTable, + "--vis", "'" + visibility + "'", "--chunkSize", Integer.toString(10000), getUsableDir()}); + assertEquals("Got non-zero return code. Stdout=" + entry.getValue(), 0, entry.getKey().intValue()); + entry = getClusterControl() + .execWithStdout( + QueryUtil.class, + new String[] {"-i", instance, "-z", keepers, "-p", passwd, "-u", user, "-t", indexTable, "--auths", auths, "--search", "--path", + "accumulo-site.xml"}); + if (ClusterType.MINI == getClusterType()) { + MiniAccumuloClusterImpl impl = (MiniAccumuloClusterImpl) cluster; + for (LogWriter writer : impl.getLogWriters()) { + writer.flush(); + } } - String result = FunctionalTestUtils.readAll(cluster, QueryUtil.class, p); - log.info("result " + result); - assertTrue(result.contains("accumulo-site.xml")); + + log.info("result " + entry.getValue()); + assertEquals(0, entry.getKey().intValue()); + assertTrue(entry.getValue().contains("accumulo-site.xml")); } @Test public void testAgeoffFilter() throws Exception { - c.tableOperations().create("filtertest"); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); is = new IteratorSetting(10, AgeOffFilter.class); AgeOffFilter.setTTL(is, 1000L); - c.tableOperations().attachIterator("filtertest", is); + c.tableOperations().attachIterator(tableName, is); UtilWaitThread.sleep(500); // let zookeeper updates propagate. - bw = c.createBatchWriter("filtertest", bwc); + bw = c.createBatchWriter(tableName, bwc); Mutation m = new Mutation("foo"); m.put("a", "b", "c"); bw.addMutation(m); bw.close(); UtilWaitThread.sleep(1000); - assertEquals(0, FunctionalTestUtils.count(c.createScanner("filtertest", Authorizations.EMPTY))); + assertEquals(0, FunctionalTestUtils.count(c.createScanner(tableName, Authorizations.EMPTY))); } @Test public void testStatsCombiner() throws Exception { - String table = "statscombinertest"; + String table = getUniqueNames(1)[0]; c.tableOperations().create(table); is = new IteratorSetting(10, StatsCombiner.class); StatsCombiner.setCombineAllColumns(is, true); @@ -213,10 +249,9 @@ public class ExamplesIT extends ConfigurableMacIT { bw.addMutation(m); bw.flush(); - - Iterator> iter = c.createScanner(table, Authorizations.EMPTY).iterator(); + Iterator> iter = c.createScanner(table, Authorizations.EMPTY).iterator(); assertTrue("Iterator had no results", iter.hasNext()); - Entry e = iter.next(); + Entry e = iter.next(); assertEquals("Results ", "1,3,4,2", e.getValue().toString()); assertFalse("Iterator had additional results", iter.hasNext()); @@ -234,21 +269,25 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testBloomFilters() throws Exception { - c.tableOperations().create("bloom_test"); - c.tableOperations().setProperty("bloom_test", Property.TABLE_BLOOM_ENABLED.getKey(), "true"); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + c.tableOperations().setProperty(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), "true"); goodExec(RandomBatchWriter.class, "--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "100000", "--min", "0", "--max", - "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", "bloom_test"); - c.tableOperations().flush("bloom_test", null, null, true); + "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", tableName); + c.tableOperations().flush(tableName, null, null, true); long diff = 0, diff2 = 0; // try the speed test a couple times in case the system is loaded with other tests for (int i = 0; i < 2; i++) { long now = System.currentTimeMillis(); goodExec(RandomBatchScanner.class, "--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max", - "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test"); + "1000000000", "--size", "50", "--scanThreads", "4", "-t", tableName); diff = System.currentTimeMillis() - now; now = System.currentTimeMillis(); - expectExec(1, RandomBatchScanner.class, "--seed", "8", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max", - "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test"); + int retCode = getClusterControl().exec( + RandomBatchScanner.class, + new String[] {"--seed", "8", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max", "1000000000", + "--size", "50", "--scanThreads", "4", "-t", tableName}); + assertEquals(1, retCode); diff2 = System.currentTimeMillis() - now; if (diff2 < diff) break; @@ -258,12 +297,14 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testShardedIndex() throws Exception { - c.tableOperations().create("shard"); - c.tableOperations().create("doc2term"); - bw = c.createBatchWriter("shard", bwc); + String[] names = getUniqueNames(3); + final String shard = names[0], index = names[1]; + c.tableOperations().create(shard); + c.tableOperations().create(index); + bw = c.createBatchWriter(shard, bwc); Index.index(30, new File(System.getProperty("user.dir") + "/src"), "\\W+", bw); bw.close(); - BatchScanner bs = c.createBatchScanner("shard", Authorizations.EMPTY, 4); + BatchScanner bs = c.createBatchScanner(shard, Authorizations.EMPTY, 4); List found = Query.query(bs, Arrays.asList("foo", "bar")); bs.close(); // should find ourselves @@ -274,20 +315,21 @@ public class ExamplesIT extends ConfigurableMacIT { } assertTrue(thisFile); // create a reverse index - c.tableOperations().create("doc2Term"); - goodExec(Reverse.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd); + goodExec(Reverse.class, "-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", getPrincipal(), "-p", passwd); // run some queries - goodExec(ContinuousQuery.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd, "--terms", - "5", "--count", "1000"); + goodExec(ContinuousQuery.class, "-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", "root", "-p", passwd, "--terms", "5", + "--count", "1000"); } @Test public void testMaxMutationConstraint() throws Exception { - c.tableOperations().create("test_ingest"); - c.tableOperations().addConstraint("test_ingest", MaxMutationSize.class.getName()); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + c.tableOperations().addConstraint(tableName, MaxMutationSize.class.getName()); TestIngest.Opts opts = new TestIngest.Opts(); opts.rows = 1; opts.cols = 1000; + opts.tableName = tableName; try { TestIngest.ingest(c, opts, bwOpts); } catch (MutationsRejectedException ex) { @@ -297,48 +339,52 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testBulkIngest() throws Exception { + String tableName = getUniqueNames(1)[0]; goodExec(GenerateTestData.class, "--start-row", "0", "--count", "10000", "--output", dir + "/tmp/input/data"); - goodExec(SetupTable.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable"); - goodExec(BulkIngestExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable", "--inputDir", dir + "/tmp/input", + goodExec(SetupTable.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName); + goodExec(BulkIngestExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName, "--inputDir", dir + "/tmp/input", "--workDir", dir + "/tmp"); } @Test public void testTeraSortAndRead() throws Exception { - String sorted = "sorted"; - goodExec(TeraSortIngest.class, "--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", sorted, "-i", instance, "-z", + String tableName = getUniqueNames(1)[0]; + goodExec(TeraSortIngest.class, "--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", tableName, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--splits", "4"); - goodExec(RegexExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", sorted, "--rowRegex", ".*999.*", "--output", dir + "/tmp/nines"); - goodExec(RowHash.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", sorted, "--column", "c:"); - goodExec(TableToFile.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", sorted, "--output", dir + "/tmp/tableFile"); + goodExec(RegexExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--rowRegex", ".*999.*", "--output", dir + + "/tmp/nines"); + goodExec(RowHash.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--column", "c:"); + goodExec(TableToFile.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--output", dir + "/tmp/tableFile"); } @Test public void testWordCount() throws Exception { - c.tableOperations().create("wordCount"); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); is = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new Text("count")))); SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING); - c.tableOperations().attachIterator("wordCount", is); + c.tableOperations().attachIterator(tableName, is); fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README"), new Path(dir + "/tmp/wc/README")); - goodExec(WordCount.class, "-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", "wordCount"); + goodExec(WordCount.class, "-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", tableName); } @Test public void testInsertWithBatchWriterAndReadData() throws Exception { - String helloBatch = "helloBatch"; - goodExec(InsertWithBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", helloBatch); - goodExec(ReadData.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", helloBatch); + String tableName = getUniqueNames(1)[0]; + goodExec(InsertWithBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName); + goodExec(ReadData.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName); } @Test public void testIsolatedScansWithInterference() throws Exception { - goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest1", "--iterations", "100000", "--isolated"); + goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", getUniqueNames(1)[0], "--iterations", "100000", + "--isolated"); } @Test public void testScansWithInterference() throws Exception { - goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest2", "--iterations", "100000"); + goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", getUniqueNames(1)[0], "--iterations", "100000"); } @Test @@ -348,39 +394,37 @@ public class ExamplesIT extends ConfigurableMacIT { @Test public void testBatchWriter() throws Exception { - c.tableOperations().create("test"); - goodExec(SequentialBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "test", "--start", "0", "--num", "100000", "--size", - "50", "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility); + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + goodExec(SequentialBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--start", "0", "--num", "100000", + "--size", "50", "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", "'" + visibility + "'"); } @Test public void testReadWriteAndDelete() throws Exception { - String test2 = "test2"; - goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", test2, "--createtable", "-c", + String tableName = getUniqueNames(1)[0]; + goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", tableName, "--createtable", "-c", "--debug"); - goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", test2, "-d", "--debug"); + goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", tableName, "-d", "--debug"); } @Test public void testRandomBatchesAndFlush() throws Exception { - String test3 = "test3"; - c.tableOperations().create(test3); - goodExec(RandomBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", test3, "--num", "100000", "--min", "0", "--max", - "100000", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility); - goodExec(RandomBatchScanner.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", test3, "--num", "10000", "--min", "0", "--max", + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + goodExec(RandomBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName, "--num", "100000", "--min", "0", "--max", + "100000", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", "'" + visibility + "'"); + goodExec(RandomBatchScanner.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName, "--num", "10000", "--min", "0", "--max", "100000", "--size", "100", "--scanThreads", "4", "--auths", auths); - goodExec(Flush.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", test3); + goodExec(Flush.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName); } - private Process goodExec(Class theClass, String... args) throws InterruptedException, IOException { - return expectExec(0, theClass, args); + private void goodExec(Class theClass, String... args) throws InterruptedException, IOException { + // We're already slurping stdout into memory (not redirecting to file). Might as well add it to error message. + Entry pair = getClusterControl().execWithStdout(theClass, args); + Assert.assertEquals("stdout=" + pair.getValue(), 0, pair.getKey().intValue()); } - private Process expectExec(int exitCode, Class theClass, String... args) throws InterruptedException, IOException { - Process p = null; - assertEquals(exitCode, (p = cluster.exec(theClass, Collections.singletonList(MapReduceIT.hadoopTmpDirArg), args)).waitFor()); - return p; - } }