accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject accumulo git commit: ACCUMULO-3871 set the default class for the mrit jar, quit out of tests that require data files unavailable under m/r, document how to invoke the m/r job, use PASS/FAIL instead of 0/1, which is a lot easier to explain
Date Fri, 19 Jun 2015 13:18:27 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 648dd8024 -> 979acacc9


ACCUMULO-3871 set the default class for the mrit jar, quit out of tests that require data
files unavailable under m/r, document how to invoke the m/r job, use PASS/FAIL instead of
0/1, which is a lot easier to explain


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/979acacc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/979acacc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/979acacc

Branch: refs/heads/master
Commit: 979acacc967e309c461190f4e1f7474296786c27
Parents: 648dd80
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Fri Jun 19 09:18:16 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Fri Jun 19 09:18:16 2015 -0400

----------------------------------------------------------------------
 .../impl/MiniAccumuloClusterImpl.java           |  1 +
 test/pom.xml                                    |  1 +
 .../accumulo/test/functional/ExamplesIT.java    | 20 +++++-
 .../accumulo/test/functional/WALSunnyDayIT.java |  4 +-
 .../test/mrit/IntegrationTestMapReduce.java     | 67 +++++++++++++++++---
 .../test/replication/CyclicReplicationIT.java   |  2 +-
 .../accumulo/test/start/KeywordStartIT.java     |  5 ++
 7 files changed, 84 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index b0794cb..4c3d6f6 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@ -398,6 +398,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster {
       conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
       conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
       conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
+      conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, "1");
       conf.set("dfs.support.append", "true");
       conf.set("dfs.datanode.synconclose", "true");
       conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 980c173..901bd6c 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -364,6 +364,7 @@
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                   <manifestEntries>
                     <Sealed>false</Sealed>
+                    <Main-Class>org.apache.accumulo.test.mrit.IntegrationTestMapReduce</Main-Class>
                   </manifestEntries>
                 </transformer>
               </transformers>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
index 9d0ce86..b43cda5 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@ -235,6 +235,10 @@ public class ExamplesIT extends AccumuloClusterHarness {
       default:
         throw new RuntimeException("Unknown cluster type");
     }
+    if (!new File(dirListDirectory).exists()) {
+      log.info("Skipping test: there's no input here");
+      return;
+    }
     // Index a directory listing on /tmp. If this is running against a standalone cluster,
we can't guarantee Accumulo source will be there.
     if (saslEnabled) {
       args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab,
"--dirTable", dirTable, "--indexTable", indexTable, "--dataTable",
@@ -253,7 +257,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
         expectedFile = "accumulo-site.xml";
         break;
       case STANDALONE:
-        // Should be in place on standalone installs (not having ot follow symlinks)
+        // Should be in place on standalone installs (not having to follow symlinks)
         expectedFile = "LICENSE";
         break;
       default:
@@ -378,12 +382,17 @@ public class ExamplesIT extends AccumuloClusterHarness {
 
   @Test
   public void testShardedIndex() throws Exception {
+    File src = new File(System.getProperty("user.dir") + "/src");
+    if (!src.exists()) {
+      log.info("Skipping test: src does not exist");
+      return;
+    }
     String[] names = getUniqueNames(3);
     final String shard = names[0], index = names[1];
     c.tableOperations().create(shard);
     c.tableOperations().create(index);
     bw = c.createBatchWriter(shard, bwc);
-    Index.index(30, new File(System.getProperty("user.dir") + "/src"), "\\W+", bw);
+    Index.index(30, src, "\\W+", bw);
     bw.close();
     BatchScanner bs = c.createBatchScanner(shard, Authorizations.EMPTY, 4);
     List<String> found = Query.query(bs, Arrays.asList("foo", "bar"));
@@ -523,7 +532,12 @@ public class ExamplesIT extends AccumuloClusterHarness {
     SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new
Text("count"))));
     SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
     c.tableOperations().attachIterator(tableName, is);
-    fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README.md"),
new Path(dir + "/tmp/wc/README.md"));
+    Path readme = new Path(new Path(System.getProperty("user.dir")).getParent(), "README.md");
+    if (!new File(readme.toString()).exists()) {
+      log.info("Not running test: README.md does not exist)");
+      return;
+    }
+    fs.copyFromLocalFile(readme, new Path(dir + "/tmp/wc/README.md"));
     String[] args;
     if (saslEnabled) {
       args = new String[] {"-i", instance, "-u", user, "--keytab", keytab, "-z", keepers,
"--input", dir + "/tmp/wc", "-t", tableName};

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
index 34d1c6d..f1d8344 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -107,7 +107,7 @@ public class WALSunnyDayIT extends ConfigurableMacBase {
     }
 
     // roll log, get a new next
-    writeSomeData(c, tableName, 1000, 50);
+    writeSomeData(c, tableName, 1001, 50);
     Map<String,Boolean> walsAfterRoll = getWals(c);
     assertEquals("should have 3 WALs after roll", 3, walsAfterRoll.size());
     assertTrue("new WALs should be a superset of the old WALs", walsAfterRoll.keySet().containsAll(wals.keySet()));
@@ -147,7 +147,7 @@ public class WALSunnyDayIT extends ConfigurableMacBase {
 
     // put some data in the WAL
     assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").waitFor());
-    verifySomeData(c, tableName, 1000 * 50 + 1);
+    verifySomeData(c, tableName, 1001 * 50 + 1);
     writeSomeData(c, tableName, 100, 100);
 
     Map<String,Boolean> walsAfterRestart = getWals(c);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
index dcc25ea..9cb807c 100644
--- a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
+++ b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
@@ -17,7 +17,10 @@
 package org.apache.accumulo.test.mrit;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
@@ -25,6 +28,7 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -40,25 +44,59 @@ import org.junit.runner.notification.RunListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Run the Integration Tests as a Map-Reduce job.
+ * <p>
+ * Each of the Integration tests takes 30s to 20m to run. Using a larger cluster, all the
tests can be run in parallel and finish much faster.
+ * <p>
+ * To run the tests, you first need a list of the tests. A simple way to get a list, is to
scan the accumulo-test jar file for them.
+ *
+ * <pre>
+ * $ jar -tf lib/accumulo-test.jar | grep IT.class | tr / . | sed -e 's/.class$//' >tests
+ * </pre>
+ *
+ * Put the list of tests into HDFS:
+ *
+ * <pre>
+ * $ hadoop fs -mkdir /tmp
+ * $ hadoop fs -put tests /tmp/tests
+ * </pre>
+ *
+ * Run the class below as a map-reduce job, giving it the lists of tests, and a place to
store the results.
+ * <pre>
+ * $ yarn jar lib/accumulo-test-mrit.jar \
+ *     -libjars lib/native/libaccumulo.so \
+ *     -Dmapreduce.map.memory.mb=4000 \
+ *     /tmp/tests /tmp/results
+ * </pre>
+ *
+ * The result is a list of IT classes that pass or fail. Those classes that fail will be
annotated with the particular test that failed within the class.
+ */
+
 public class IntegrationTestMapReduce extends Configured implements Tool {
 
   private static final Logger log = LoggerFactory.getLogger(IntegrationTestMapReduce.class);
 
-  public static class TestMapper extends Mapper<LongWritable,Text,IntWritable,Text>
{
+  public static class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
+
+    static final Text FAIL = new Text("FAIL");
+    static final Text PASS = new Text("PASS");
+    static final Text ERROR = new Text("ERROR");
 
     @Override
-    protected void map(LongWritable key, Text value, final Mapper<LongWritable,Text,IntWritable,Text>.Context
context) throws IOException, InterruptedException {
+    protected void map(LongWritable key, Text value, final Mapper<LongWritable,Text,Text,Text>.Context
context) throws IOException, InterruptedException {
       String className = value.toString();
       if (className.trim().isEmpty()) {
         return;
       }
       log.info("Running test {}", className);
+      final List<String> failures = new ArrayList<>();
       Class<? extends Object> test = null;
       try {
         test = Class.forName(className);
       } catch (ClassNotFoundException e) {
         log.debug("Error finding class {}", className, e);
-        context.write(new IntWritable(-1), new Text(e.toString()));
+        context.write(ERROR, new Text(e.toString()));
         return;
       }
       JUnitCore core = new JUnitCore();
@@ -79,6 +117,7 @@ public class IntegrationTestMapReduce extends Configured implements Tool
{
         @Override
         public void testFailure(Failure failure) throws Exception {
           log.info("Test failed: {}", failure.getDescription(), failure.getException());
+          failures.add(failure.getDescription().getMethodName());
           context.progress();
         }
 
@@ -88,10 +127,10 @@ public class IntegrationTestMapReduce extends Configured implements Tool
{
         Result result = core.run(test);
         if (result.wasSuccessful()) {
           log.info("{} was successful", className);
-          context.write(new IntWritable(0), value);
+          context.write(PASS, value);
         } else {
           log.info("{} failed", className);
-          context.write(new IntWritable(1), value);
+          context.write(FAIL, new Text(className + "(" + StringUtils.join(failures, ", ")
+ ")"));
         }
       } catch (Exception e) {
         // most likely JUnit issues, like no tests to run
@@ -100,14 +139,14 @@ public class IntegrationTestMapReduce extends Configured implements
Tool {
     }
   }
 
-  public static class TestReducer extends Reducer<IntWritable,Text,IntWritable,Text>
{
+  public static class TestReducer extends Reducer<Text,Text,Text,Text> {
 
     @Override
-    protected void reduce(IntWritable code, Iterable<Text> tests, Reducer<IntWritable,Text,IntWritable,Text>.Context
context) throws IOException,
-        InterruptedException {
-      StringBuffer result = new StringBuffer();
+    protected void reduce(Text code, Iterable<Text> tests, Reducer<Text,Text,Text,Text>.Context
context) throws IOException, InterruptedException {
+      StringBuffer result = new StringBuffer("\n");
       for (Text test : tests) {
-        result.append(test);
+        result.append("   ");
+        result.append(test.toString());
         result.append("\n");
       }
       context.write(code, new Text(result.toString()));
@@ -119,9 +158,17 @@ public class IntegrationTestMapReduce extends Configured implements Tool
{
     // read a list of tests from the input, and print out the results
     if (args.length != 2) {
       System.err.println("Wrong number of args: <input> <output>");
+      return 1;
     }
     Configuration conf = getConf();
     Job job = Job.getInstance(conf, "accumulo integration test runner");
+
+    // no need to run a test multiple times
+    job.setSpeculativeExecution(false);
+
+    // hadoop puts an ancient version of jline on the classpath
+    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
+
     // read one line at a time
     job.setInputFormatClass(NLineInputFormat.class);
     conf.setInt(NLineInputFormat.LINES_PER_MAP, 1);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java
b/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java
index 3a1d413..3abdb0e 100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java
@@ -79,7 +79,7 @@ public class CyclicReplicationIT {
       log.warn("Could not parse timeout.factor, not scaling timeout");
     }
 
-    return new Timeout(scalingFactor * 5 * 60 * 1000);
+    return new Timeout(scalingFactor * 10 * 60 * 1000);
   }
 
   @Rule

http://git-wip-us.apache.org/repos/asf/accumulo/blob/979acacc/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index f7f250a..9201bea 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
@@ -92,6 +93,10 @@ public class KeywordStartIT {
   // Note: this test may fail in Eclipse, if the services files haven't been generated by
the AutoService annotation processor
   @Test
   public void testExpectedClasses() throws IOException {
+    if (!new File(System.getProperty("user.dir") + "/src").exists()) {
+      log.info("Ignoring test: probably running under map-reduce");
+      return;
+    }
     TreeMap<String,Class<? extends KeywordExecutable>> expectSet = new TreeMap<>();
     expectSet.put("admin", Admin.class);
     expectSet.put("classpath", Classpath.class);


Mime
View raw message