accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject [1/5] git commit: ACCUMULO-2298 chunking up ExamplesIT
Date Sat, 01 Feb 2014 02:31:33 GMT
Updated Branches:
  refs/heads/master 7f5bd5f5a -> 6569b2c94


ACCUMULO-2298 chunking up ExamplesIT


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/be152d85
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/be152d85
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/be152d85

Branch: refs/heads/master
Commit: be152d8505e24591c3dac484787b2dfc9f6c0058
Parents: b178784
Author: John Vines <vines@apache.org>
Authored: Fri Jan 31 15:17:08 2014 -0500
Committer: John Vines <vines@apache.org>
Committed: Fri Jan 31 15:17:08 2014 -0500

----------------------------------------------------------------------
 .../accumulo/test/functional/AbstractMacIT.java |   2 +-
 .../accumulo/test/functional/ExamplesIT.java    | 205 +++++++++++++------
 2 files changed, 138 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/be152d85/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java
index ecfcf59..aab3edb 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java
@@ -65,7 +65,7 @@ public abstract class AbstractMacIT {
     return testDir;
   }
 
-  protected File createTestDir(String name) {
+  protected static File createTestDir(String name) {
     File baseDir = new File(System.getProperty("user.dir") + "/target/mini-tests");
     baseDir.mkdirs();
     if (name == null)

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be152d85/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
index 559ae67..6ac0cd3 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@ -29,6 +29,8 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchScanner;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
@@ -71,6 +73,7 @@ import org.apache.accumulo.examples.simple.shard.Index;
 import org.apache.accumulo.examples.simple.shard.Query;
 import org.apache.accumulo.examples.simple.shard.Reverse;
 import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 import org.apache.accumulo.server.util.Admin;
@@ -79,37 +82,63 @@ import org.apache.accumulo.tracer.TraceServer;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-public class ExamplesIT extends ConfigurableMacIT {
-
+public class ExamplesIT extends AbstractMacIT {
   BatchWriterOpts bwOpts = new BatchWriterOpts();
 
-  @Override
-  public void configure(MiniAccumuloConfigImpl cfg) {
+  static Connector c;
+  static String instance;
+  static String keepers;
+  static String user = "root";
+  static String passwd;
+  String visibility = "A|B";
+  static String auths = "A,B";
+  BatchWriterConfig bwc = new BatchWriterConfig();
+  BatchWriter bw;
+  IteratorSetting is;
+  static String dir;
+  static FileSystem fs;
+  private static MiniAccumuloClusterImpl cluster;
+
+  @BeforeClass
+  public static void before() throws Exception {
+    MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(createTestDir(ExamplesIT.class.getName()),
AbstractMacIT.ROOT_PASSWORD);
+    cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath());
     cfg.setDefaultMemory(cfg.getDefaultMemory() * 2, MemoryUnit.BYTE);
+
+    cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
+    configureForEnvironment(cfg, ExamplesIT.class, createSharedTestDir(ExamplesIT.class.getName()
+ "-ssl"));
+    cluster = new MiniAccumuloClusterImpl(cfg);
+    cluster.start();
+
+    passwd = AbstractMacIT.ROOT_PASSWORD;
+    c = cluster.getConnector("root", ROOT_PASSWORD);
+    fs = FileSystem.get(CachedConfiguration.getInstance());
+    instance = c.getInstance().getInstanceName();
+    keepers = c.getInstance().getZooKeepers();
+    dir = cluster.getConfig().getDir().getAbsolutePath();
+
+    c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(",")));
   }
 
-  @Test(timeout = 15 * 60 * 1000)
-  public void test() throws Exception {
-    Connector c = getConnector();
-    String instance = c.getInstance().getInstanceName();
-    String keepers = c.getInstance().getZooKeepers();
-    String user = "root";
-    String passwd = ROOT_PASSWORD;
-    String visibility = "A|B";
-    String auths = "A,B";
-    BatchWriterConfig bwc = new BatchWriterConfig();
-    BatchWriter bw;
-    IteratorSetting is;
-    String dir = cluster.getConfig().getDir().getAbsolutePath();
-    FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+  @Override
+  public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+    return cluster.getConnector("root", ROOT_PASSWORD);
+  }
+
+  @Override
+  public String rootPath() {
+    return cluster.getConfig().getDir().getAbsolutePath();
+  }
 
+  @Test(timeout = 15 * 1000)
+  public void testTrace() throws Exception {
     Process trace = exec(TraceServer.class);
     while (!c.tableOperations().exists("trace"))
       UtilWaitThread.sleep(500);
-
-    log.info("trace example");
     Process p = goodExec(TracingExample.class, "-i", instance, "-z", keepers, "-u", user,
"-p", passwd, "-C", "-D", "-c");
     for (LogWriter writer : cluster.getLogWriters()) {
       writer.flush();
@@ -126,21 +155,28 @@ public class ExamplesIT extends ConfigurableMacIT {
     result = FunctionalTestUtils.readAll(cluster, TraceDumpExample.class, p);
     assertTrue(result.contains("myHost@myApp"));
     trace.destroy();
+  }
 
-    log.info("testing dirlist example (a little)");
-    c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(",")));
+  private Process exec(Class<TraceServer> class1) throws IOException {
+    return cluster.exec(class1);
+  }
+
+  @Test(timeout = 45 * 1000)
+  public void testDirList() throws Exception {
     goodExec(Ingest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable",
"dirTable", "--indexTable", "indexTable", "--dataTable",
         "dataTable", "--vis", visibility, "--chunkSize", 10000 + "", cluster.getConfig().getDir().getAbsolutePath());
-    p = goodExec(QueryUtil.class, "-i", instance, "-z", keepers, "-p", passwd, "-u", user,
"-t", "indexTable", "--auths", auths, "--search", "--path",
+    Process p = goodExec(QueryUtil.class, "-i", instance, "-z", keepers, "-p", passwd, "-u",
user, "-t", "indexTable", "--auths", auths, "--search", "--path",
         "accumulo-site.xml");
     for (LogWriter writer : cluster.getLogWriters()) {
       writer.flush();
     }
-    result = FunctionalTestUtils.readAll(cluster, QueryUtil.class, p);
+    String result = FunctionalTestUtils.readAll(cluster, QueryUtil.class, p);
     System.out.println("result " + result);
     assertTrue(result.contains("accumulo-site.xml"));
+  }
 
-    log.info("Testing ageoff filtering");
+  @Test(timeout = 5 * 1000)
+  public void testAgeoffFilter() throws Exception {
     c.tableOperations().create("filtertest");
     is = new IteratorSetting(10, AgeOffFilter.class);
     AgeOffFilter.setTTL(is, 1000L);
@@ -150,35 +186,39 @@ public class ExamplesIT extends ConfigurableMacIT {
     m.put("a", "b", "c");
     bw.addMutation(m);
     UtilWaitThread.sleep(1000);
-    count = 0;
+    int count = 0;
     for (@SuppressWarnings("unused")
     Entry<Key,Value> line : c.createScanner("filtertest", Authorizations.EMPTY))
       count++;
     assertEquals(0, count);
+  }
 
-    log.info("Testing bloom filters are fast for missing data");
+  @Test(timeout = 20 * 1000)
+  public void testBloomFilters() throws Exception {
     c.tableOperations().create("bloom_test");
     c.tableOperations().setProperty("bloom_test", Property.TABLE_BLOOM_ENABLED.getKey(),
"true");
-    goodExec(RandomBatchWriter.class, "--seed", "7", "-i", instance, "-z", keepers, "-u",
user, "-p", ROOT_PASSWORD, "--num", "100000", "--min", "0", "--max",
+    goodExec(RandomBatchWriter.class, "--seed", "7", "-i", instance, "-z", keepers, "-u",
user, "-p", passwd, "--num", "100000", "--min", "0", "--max",
         "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads",
"3", "-t", "bloom_test");
     c.tableOperations().flush("bloom_test", null, null, true);
     long diff = 0, diff2 = 0;
     // try the speed test a couple times in case the system is loaded with other tests
     for (int i = 0; i < 2; i++) {
       long now = System.currentTimeMillis();
-      goodExec(RandomBatchScanner.class, "--seed", "7", "-i", instance, "-z", keepers, "-u",
user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0",
-          "--max", "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test");
+      goodExec(RandomBatchScanner.class, "--seed", "7", "-i", instance, "-z", keepers, "-u",
user, "-p", passwd, "--num", "10000", "--min", "0", "--max",
+          "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test");
       diff = System.currentTimeMillis() - now;
       now = System.currentTimeMillis();
-      expectExec(1, RandomBatchScanner.class, "--seed", "8", "-i", instance, "-z", keepers,
"-u", user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0",
-          "--max", "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test");
+      expectExec(1, RandomBatchScanner.class, "--seed", "8", "-i", instance, "-z", keepers,
"-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max",
+          "1000000000", "--size", "50", "--scanThreads", "4", "-t", "bloom_test");
       diff2 = System.currentTimeMillis() - now;
       if (diff2 < diff)
         break;
     }
     assertTrue(diff2 < diff);
+  }
 
-    log.info("Creating a sharded index of the accumulo java files");
+  @Test(timeout = 2 * 60 * 1000)
+  public void testShardedIndex() throws Exception {
     c.tableOperations().create("shard");
     c.tableOperations().create("doc2term");
     bw = c.createBatchWriter("shard", bwc);
@@ -200,8 +240,10 @@ public class ExamplesIT extends ConfigurableMacIT {
     // run some queries
     goodExec(ContinuousQuery.class, "-i", instance, "-z", keepers, "--shardTable", "shard",
"--doc2Term", "doc2Term", "-u", "root", "-p", passwd, "--terms",
         "5", "--count", "1000");
+  }
 
-    log.info("Testing MaxMutation constraint");
+  @Test(timeout = 2 * 1000)
+  public void testMaxMutationConstraint() throws Exception {
     c.tableOperations().create("test_ingest");
     c.tableOperations().addConstraint("test_ingest", MaxMutationSize.class.getName());
     TestIngest.Opts opts = new TestIngest.Opts();
@@ -212,25 +254,28 @@ public class ExamplesIT extends ConfigurableMacIT {
     } catch (MutationsRejectedException ex) {
       assertEquals(1, ex.getConstraintViolationSummaries().size());
     }
+  }
 
-    log.info("Starting bulk ingest example");
+  @Test(timeout = 20 * 1000)
+  public void testBulkIngest() throws Exception {
     goodExec(GenerateTestData.class, "--start-row", "0", "--count", "10000", "--output",
dir + "/tmp/input/data");
     goodExec(SetupTable.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table",
"bulkTable");
     goodExec(BulkIngestExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--table", "bulkTable", "--inputDir", dir + "/tmp/input",
         "--workDir", dir + "/tmp");
+  }
 
-    log.info("Running TeraSortIngest example");
-    goodExec(TeraSortIngest.class, "--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10",
"-nv", "10", "-xv", "10", "-t", "sorted", "-i", instance, "-z",
+  @Test(timeout = 1 * 60 * 1000)
+  public void testTeraSortAndRead() throws Exception {
+    String sorted = "sorted";
+    goodExec(TeraSortIngest.class, "--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10",
"-nv", "10", "-xv", "10", "-t", sorted, "-i", instance, "-z",
         keepers, "-u", user, "-p", passwd, "--splits", "4");
-    log.info("Running Regex example");
-    goodExec(RegexExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", "sorted", "--rowRegex", ".*999.*", "--output", dir
-        + "/tmp/nines");
-    log.info("Running RowHash example");
-    goodExec(RowHash.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t",
"sorted", "--column", "c:");
-    log.info("Running TableToFile example");
-    goodExec(TableToFile.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", "sorted", "--output", dir + "/tmp/tableFile");
-
-    log.info("Running word count example");
+    goodExec(RegexExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", sorted, "--rowRegex", ".*999.*", "--output", dir + "/tmp/nines");
+    goodExec(RowHash.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t",
sorted, "--column", "c:");
+    goodExec(TableToFile.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", sorted, "--output", dir + "/tmp/tableFile");
+  }
+
+  @Test(timeout = 10 * 1000)
+  public void testWordCount() throws Exception {
     c.tableOperations().create("wordCount");
     is = new IteratorSetting(10, SummingCombiner.class);
     SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new
Text("count"))));
@@ -238,45 +283,69 @@ public class ExamplesIT extends ConfigurableMacIT {
     c.tableOperations().attachIterator("wordCount", is);
     fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README"),
new Path(dir + "/tmp/wc/README"));
     goodExec(WordCount.class, "-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input",
dir + "/tmp/wc", "-t", "wordCount");
+  }
 
-    log.info("Inserting data with a batch writer");
-    goodExec(InsertWithBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p",
passwd, "-t", "helloBatch");
-    log.info("Reading data");
-    goodExec(ReadData.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t",
"helloBatch");
-    log.info("Running isolated scans");
+  @Test(timeout = 30 * 1000)
+  public void testInsertWithBatchWriterAndReadData() throws Exception {
+    String helloBatch = "helloBatch";
+    goodExec(InsertWithBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p",
passwd, "-t", helloBatch);
+    goodExec(ReadData.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t",
helloBatch);
+  }
+
+  @Test(timeout = 90 * 1000)
+  public void testIsolatedScansWithInterference() throws Exception {
     goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", "itest1", "--iterations", "100000", "--isolated");
-    log.info("Running scans without isolation");
+  }
+
+  @Test(timeout = 75 * 1000)
+  public void testScansWithInterference() throws Exception {
     goodExec(InterferenceTest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"-t", "itest2", "--iterations", "100000");
-    log.info("Performing some row operations");
+  }
+
+  @Test(timeout = 5 * 1000)
+  public void testRowOperations() throws Exception {
     goodExec(RowOperations.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd);
-    log.info("Using the batch writer");
+  }
+
+  @Test(timeout = 6 * 1000)
+  public void testBatchWriter() throws Exception {
     c.tableOperations().create("test");
     goodExec(SequentialBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p",
passwd, "-t", "test", "--start", "0", "--num", "100000", "--size",
         "50", "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4",
"--vis", visibility);
 
-    log.info("Reading and writing some data");
-    goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--auths", auths, "--table", "test2", "--createtable", "-c",
+  }
+
+  @Test(timeout = 12 * 1000)
+  public void testReadWriteAndDelete() throws Exception {
+    String test2 = "test2";
+    goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--auths", auths, "--table", test2, "--createtable", "-c",
         "--debug");
-    log.info("Deleting some data");
-    goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--auths", auths, "--table", "test2", "-d", "--debug");
-    log.info("Writing some data with the batch writer");
-    c.tableOperations().create("test3");
-    goodExec(RandomBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--table", "test3", "--num", "100000", "--min", "0", "--max",
+    goodExec(ReadWriteExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--auths", auths, "--table", test2, "-d", "--debug");
+
+  }
+
+  @Test(timeout = 20 * 1000)
+  public void testRandomBatchesAndFlush() throws Exception {
+    String test3 = "test3";
+    c.tableOperations().create(test3);
+    goodExec(RandomBatchWriter.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--table", test3, "--num", "100000", "--min", "0", "--max",
         "100000", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000",
"--batchThreads", "4", "--vis", visibility);
-    log.info("Reading some data with the batch scanner");
-    goodExec(RandomBatchScanner.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--table", "test3", "--num", "10000", "--min", "0", "--max",
+    goodExec(RandomBatchScanner.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd,
"--table", test3, "--num", "10000", "--min", "0", "--max",
         "100000", "--size", "100", "--scanThreads", "4", "--auths", auths);
-    log.info("Running an example table operation (Flush)");
-    goodExec(Flush.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table",
"test3");
-    goodExec(Admin.class, "stopAll");
+    goodExec(Flush.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table",
test3);
+  }
 
+  @AfterClass
+  public static void stop() throws Exception {
+    goodExec(Admin.class, "stopAll");
+    cleanUp(cluster);
   }
 
-  private Process goodExec(Class<?> theClass, String... args) throws InterruptedException,
IOException {
+  private static Process goodExec(Class<?> theClass, String... args) throws InterruptedException,
IOException {
     return expectExec(0, theClass, args);
   }
 
-  private Process expectExec(int exitCode, Class<?> theClass, String... args) throws
InterruptedException, IOException {
+  private static Process expectExec(int exitCode, Class<?> theClass, String... args)
throws InterruptedException, IOException {
     Process p = null;
     assertEquals(exitCode, (p = cluster.exec(theClass, Collections.singletonList(MapReduceIT.hadoopTmpDirArg),
args)).waitFor());
     return p;


Mime
View raw message