accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject accumulo git commit: ACCUMULO-3423 tightended some timings, added more m/r options required to run the tests, figured out how to get distcp to work when already running in m/r, used Assume methods to bail on tests that fail in the m/r environment
Date Fri, 19 Jun 2015 20:46:44 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 979acacc9 -> b7f88f9df


ACCUMULO-3423 tightended some timings, added more m/r options required to run the tests, figured
out how to get distcp to work when already running in m/r, used Assume methods to bail on
tests that fail in the m/r environment


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b7f88f9d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b7f88f9d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b7f88f9d

Branch: refs/heads/master
Commit: b7f88f9dfb3d244d7b1b684b4c225514b868e9e0
Parents: 979acac
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Fri Jun 19 16:46:37 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Fri Jun 19 16:46:37 2015 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/test/DumpConfigIT.java  |  8 +++----
 .../org/apache/accumulo/test/ShellServerIT.java |  8 +++----
 .../accumulo/test/functional/ExamplesIT.java    | 18 ++++++----------
 .../test/functional/MetadataMaxFilesIT.java     | 17 +++++++--------
 .../accumulo/test/functional/ReadWriteIT.java   |  4 ++--
 .../test/mrit/IntegrationTestMapReduce.java     | 22 ++++++++++++--------
 .../accumulo/test/start/KeywordStartIT.java     |  6 ++----
 7 files changed, 39 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
index 5cc37a5..01d873e 100644
--- a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java
@@ -31,15 +31,11 @@ import org.apache.accumulo.server.util.Admin;
 import org.apache.accumulo.test.functional.ConfigurableMacBase;
 import org.apache.accumulo.test.functional.FunctionalTestUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class DumpConfigIT extends ConfigurableMacBase {
 
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir")
+ "/target"));
-
   @Override
   public int defaultTimeoutSeconds() {
     return 2 * 60;
@@ -52,6 +48,10 @@ public class DumpConfigIT extends ConfigurableMacBase {
 
   @Test
   public void test() throws Exception {
+    File target = new File(System.getProperty("user.dir"), "target");
+    target.mkdirs();
+    TemporaryFolder folder = new TemporaryFolder(target);
+    folder.create();
     File siteFileBackup = new File(folder.getRoot(), "accumulo-site.xml.bak");
     assertFalse(siteFileBackup.exists());
     assertEquals(0, exec(Admin.class, new String[] {"dumpConfig", "-a", "-d", folder.getRoot().getPath()}).waitFor());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index e01de42..cf22df5 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -315,7 +315,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
     String exportUri = "file://" + exportDir.toString();
     String localTmp = "file://" + new File(rootPath, "ShellServerIT.tmp").toString();
     ts.exec("exporttable -t " + table + " " + exportUri, true);
-    DistCp cp = newDistCp();
+    DistCp cp = newDistCp(new Configuration(false));
     String import_ = "file://" + new File(rootPath, "ShellServerIT.import").toString();
     if (getCluster().getClientConfig().getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(),
false)) {
       // DistCp bugs out trying to get a fs delegation token to perform the cp. Just copy
it ourselves by hand.
@@ -355,7 +355,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
     ts.exec("deletetable -f " + table2, true);
   }
 
-  private DistCp newDistCp() {
+  private DistCp newDistCp(Configuration conf) {
     try {
       @SuppressWarnings("unchecked")
       Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
@@ -363,9 +363,9 @@ public class ShellServerIT extends SharedMiniClusterBase {
         Class<?>[] parameterTypes = constructor.getParameterTypes();
         if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class))
{
           if (parameterTypes.length == 1) {
-            return constructor.newInstance(new Configuration());
+            return constructor.newInstance(conf);
           } else if (parameterTypes.length == 2) {
-            return constructor.newInstance(new Configuration(), null);
+            return constructor.newInstance(conf, null);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
index b43cda5..671a6bb 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@ -20,6 +20,7 @@ import static com.google.common.base.Charsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
 import java.io.IOException;
@@ -95,7 +96,6 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.util.Tool;
 import org.junit.After;
 import org.junit.Assert;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -235,10 +235,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
       default:
         throw new RuntimeException("Unknown cluster type");
     }
-    if (!new File(dirListDirectory).exists()) {
-      log.info("Skipping test: there's no input here");
-      return;
-    }
+    assumeTrue(new File(dirListDirectory).exists());
     // Index a directory listing on /tmp. If this is running against a standalone cluster,
we can't guarantee Accumulo source will be there.
     if (saslEnabled) {
       args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab,
"--dirTable", dirTable, "--indexTable", indexTable, "--dataTable",
@@ -383,10 +380,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
   @Test
   public void testShardedIndex() throws Exception {
     File src = new File(System.getProperty("user.dir") + "/src");
-    if (!src.exists()) {
-      log.info("Skipping test: src does not exist");
-      return;
-    }
+    assumeTrue(src.exists());
     String[] names = getUniqueNames(3);
     final String shard = names[0], index = names[1];
     c.tableOperations().create(shard);
@@ -449,7 +443,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
   @Test
   public void testBulkIngest() throws Exception {
     // TODO Figure out a way to run M/R with Kerberos
-    Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+    assumeTrue(getAdminToken() instanceof PasswordToken);
     String tableName = getUniqueNames(1)[0];
     FileSystem fs = getFileSystem();
     Path p = new Path(dir, "tmp");
@@ -482,7 +476,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
   @Test
   public void testTeraSortAndRead() throws Exception {
     // TODO Figure out a way to run M/R with Kerberos
-    Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+    assumeTrue(getAdminToken() instanceof PasswordToken);
     String tableName = getUniqueNames(1)[0];
     String[] args;
     if (saslEnabled) {
@@ -525,7 +519,7 @@ public class ExamplesIT extends AccumuloClusterHarness {
   @Test
   public void testWordCount() throws Exception {
     // TODO Figure out a way to run M/R with Kerberos
-    Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+    assumeTrue(getAdminToken() instanceof PasswordToken);
     String tableName = getUniqueNames(1)[0];
     c.tableOperations().create(tableName);
     is = new IteratorSetting(10, SummingCombiner.class);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index 086dd1a..977a659 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@ -18,8 +18,6 @@ package org.apache.accumulo.test.functional;
 
 import static org.junit.Assert.assertEquals;
 
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -49,10 +47,9 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
 
   @Override
   public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-    Map<String,String> siteConfig = new HashMap<String,String>();
-    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
-    siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
-    cfg.setSiteConfig(siteConfig);
+    cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+    cfg.setProperty(Property.TSERV_SCAN_MAX_OPENFILES, "10");
+    cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "100");
     hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
   }
 
@@ -69,6 +66,8 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
       splits.add(new Text(String.format("%03d", i)));
     }
     c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(),
"10000");
+    // propagation time
+    UtilWaitThread.sleep(5 * 1000);
     for (int i = 0; i < 5; i++) {
       String tableName = "table" + i;
       log.info("Creating " + tableName);
@@ -79,22 +78,21 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
       c.tableOperations().flush(MetadataTable.NAME, null, null, true);
       c.tableOperations().flush(RootTable.NAME, null, null, true);
     }
-    UtilWaitThread.sleep(20 * 1000);
     log.info("shutting down");
     assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
     cluster.stop();
     log.info("starting up");
     cluster.start();
 
-    UtilWaitThread.sleep(30 * 1000);
+    Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
 
     while (true) {
       MasterMonitorInfo stats = null;
-      Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
       Client client = null;
       try {
         ClientContext context = new ClientContext(c.getInstance(), creds, getClientConfig());
         client = MasterClient.getConnectionWithRetry(context);
+        log.info("Fetching stats");
         stats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
       } finally {
         if (client != null)
@@ -108,6 +106,7 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase {
           tablets += entry.getValue().onlineTablets;
         }
       }
+      log.info("Online tablets " + tablets);
       if (tablets == 5005)
         break;
       UtilWaitThread.sleep(1000);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index d048071..485d6d2 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -107,7 +107,7 @@ public class ReadWriteIT extends AccumuloClusterHarness {
 
   private static final Logger log = LoggerFactory.getLogger(ReadWriteIT.class);
 
-  static final int ROWS = 200000;
+  static final int ROWS = 100000;
   static final int COLS = 1;
   static final String COLF = "colf";
 
@@ -443,7 +443,7 @@ public class ReadWriteIT extends AccumuloClusterHarness {
 
   @Test
   public void localityGroupChange() throws Exception {
-    // Make changes to locality groups and ensure nothing is lostssh
+    // Make changes to locality groups and ensure nothing is lost
     final Connector connector = getConnector();
     String table = getUniqueNames(1)[0];
     TableOperations to = connector.tableOperations();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
index 9cb807c..2573f8e 100644
--- a/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
+++ b/test/src/main/java/org/apache/accumulo/test/mrit/IntegrationTestMapReduce.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
@@ -63,11 +62,9 @@ import org.slf4j.LoggerFactory;
  * </pre>
  *
  * Run the class below as a map-reduce job, giving it the lists of tests, and a place to
store the results.
+ *
  * <pre>
- * $ yarn jar lib/accumulo-test-mrit.jar \
- *     -libjars lib/native/libaccumulo.so \
- *     -Dmapreduce.map.memory.mb=4000 \
- *     /tmp/tests /tmp/results
+ * $ yarn jar lib/accumulo-test-mrit.jar -libjars lib/native/libaccumulo.so /tmp/tests /tmp/results
  * </pre>
  *
  * The result is a list of IT classes that pass or fail. Those classes that fail will be
annotated with the particular test that failed within the class.
@@ -162,16 +159,23 @@ public class IntegrationTestMapReduce extends Configured implements
Tool {
     }
     Configuration conf = getConf();
     Job job = Job.getInstance(conf, "accumulo integration test runner");
+    conf = job.getConfiguration();
 
-    // no need to run a test multiple times
-    job.setSpeculativeExecution(false);
+    // some tests take more than 10 minutes
+    conf.setLong(MRJobConfig.TASK_TIMEOUT, 20 * 60 * 1000);
+
+    // minicluster uses a lot of ram
+    conf.setInt(MRJobConfig.MAP_MEMORY_MB, 4000);
 
     // hadoop puts an ancient version of jline on the classpath
     conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
 
+    // no need to run a test multiple times
+    job.setSpeculativeExecution(false);
+
     // read one line at a time
     job.setInputFormatClass(NLineInputFormat.class);
-    conf.setInt(NLineInputFormat.LINES_PER_MAP, 1);
+    NLineInputFormat.setNumLinesPerSplit(job, 1);
 
     // run the test
     job.setJarByClass(IntegrationTestMapReduce.class);
@@ -179,7 +183,7 @@ public class IntegrationTestMapReduce extends Configured implements Tool
{
 
     // group test by result code
     job.setReducerClass(TestReducer.class);
-    job.setOutputKeyClass(IntWritable.class);
+    job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(Text.class);
 
     FileInputFormat.addInputPath(job, new Path(args[0]));

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7f88f9d/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index 9201bea..40f6822 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.start;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
 import java.io.IOException;
@@ -93,10 +94,7 @@ public class KeywordStartIT {
   // Note: this test may fail in Eclipse, if the services files haven't been generated by
the AutoService annotation processor
   @Test
   public void testExpectedClasses() throws IOException {
-    if (!new File(System.getProperty("user.dir") + "/src").exists()) {
-      log.info("Ignoring test: probably running under map-reduce");
-      return;
-    }
+    assumeTrue(new File(System.getProperty("user.dir") + "/src").exists());
     TreeMap<String,Class<? extends KeywordExecutable>> expectSet = new TreeMap<>();
     expectSet.put("admin", Admin.class);
     expectSet.put("classpath", Classpath.class);


Mime
View raw message