accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject git commit: ACCUMULO-1823 more tweaks to help stabilize the ITs, get more debugging for post-test analysis
Date Tue, 29 Oct 2013 18:22:03 GMT
Updated Branches:
  refs/heads/master 219560a9a -> 135e67b68


ACCUMULO-1823 more tweaks to help stabilize the ITs, get more debugging for post-test analysis


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/135e67b6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/135e67b6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/135e67b6

Branch: refs/heads/master
Commit: 135e67b68592f0d1c7ca69bac318a7ad3ed55831
Parents: 219560a
Author: Eric Newton <eric.newton@gmail.com>
Authored: Tue Oct 29 13:39:16 2013 -0400
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Tue Oct 29 14:22:16 2013 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java  |  7 ++++++-
 .../minicluster/MiniAccumuloCluster.java         |  2 ++
 .../apache/accumulo/server/master/Master.java    |  2 +-
 .../test/functional/GarbageCollectorIT.java      |  1 +
 .../accumulo/test/functional/RestartIT.java      | 19 ++++++++++---------
 .../test/functional/RestartStressIT.java         |  4 ++--
 .../test/functional/SplitRecoveryIT.java         |  2 +-
 test/src/test/resources/log4j.properties         |  5 +++--
 8 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 134a037..bee4265 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.conf;
 
 import java.io.File;
 import java.lang.annotation.Annotation;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -573,6 +574,9 @@ public enum Property {
         || key.equals(Property.TABLE_LOAD_BALANCER.getKey());
   }
 
+  // This is not a cache for loaded classes, just a way to avoid spamming the debug log
+  static Map<String, Class<? extends Object>> loaded = Collections.synchronizedMap(new
HashMap<String, Class<? extends Object>>()); 
+
   public static <T> T createInstanceFromPropertyName(AccumuloConfiguration conf, Property
property, Class<T> base, T defaultInstance) {
     String clazzName = conf.get(property);
     T instance = null;
@@ -580,7 +584,8 @@ public enum Property {
     try {
       Class<? extends T> clazz = AccumuloVFSClassLoader.loadClass(clazzName, base);
       instance = clazz.newInstance();
-      log.debug("Loaded class : " + clazzName);
+      if (loaded.put(clazzName, clazz) != clazz)
+        log.debug("Loaded class : " + clazzName);
     } catch (Exception e) {
       log.warn("Failed to load class ", e);
     }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 1c7ad74..a0cc8b1 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -240,6 +240,8 @@ public class MiniAccumuloCluster {
     builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
+    // hadoop-2.2 puts error messages in the logs if this is not set
+    builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
 
     Process process = builder.start();
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java
index c029ae5..28da13d 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -183,7 +183,7 @@ public class Master implements LiveTServerSet.Listener, TableObserver,
CurrentSt
   final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
   final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
   final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
-  final static int MAX_TSERVER_WORK_CHUNK = 5 * ONE_SECOND;
+  final static int MAX_TSERVER_WORK_CHUNK = 5000;
   final private static int MAX_BAD_STATUS_COUNT = 3;
   
   final VolumeManager fs;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 7b9282c..9cdd9ab 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -60,6 +60,7 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
     settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
     settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
     cfg.setSiteConfig(settings);
+    cfg.useMiniDFS();
   }
 
   @Test(timeout = 4 * 60 * 1000)

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index de29d3b..79a895b 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@ -53,7 +53,11 @@ public class RestartIT extends ConfigurableMacIT {
 
   private static final ScannerOpts SOPTS = new ScannerOpts();
   private static final VerifyIngest.Opts VOPTS = new VerifyIngest.Opts();
+  private static final TestIngest.Opts OPTS = new TestIngest.Opts();
   private static final BatchWriterOpts BWOPTS = new BatchWriterOpts();
+  static {
+    OPTS.rows = VOPTS.rows = 10 * 1000;
+  }
   
   @Test(timeout = 2 * 60 * 1000)
   public void restartMaster() throws Exception {
@@ -61,7 +65,7 @@ public class RestartIT extends ConfigurableMacIT {
     c.tableOperations().create("test_ingest");
     Process ingest = cluster.exec(TestIngest.class, 
         "-u", "root", "-p", ROOT_PASSWORD, 
-        "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers());
+        "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "--rows", ""+OPTS.rows);
     for (ProcessReference master : cluster.getProcesses().get(ServerType.MASTER)) {
       cluster.killProcess(ServerType.MASTER, master);
     }
@@ -75,8 +79,7 @@ public class RestartIT extends ConfigurableMacIT {
   public void restartMasterRecovery() throws Exception {
     Connector c = getConnector();
     c.tableOperations().create("test_ingest");
-    TestIngest.Opts opts = new TestIngest.Opts();
-    TestIngest.ingest(c, opts, BWOPTS);
+    TestIngest.ingest(c, OPTS, BWOPTS);
     for (Entry<ServerType,Collection<ProcessReference>> entry : cluster.getProcesses().entrySet())
{
       for (ProcessReference proc : entry.getValue()) {
         cluster.killProcess(entry.getKey(), proc);
@@ -98,7 +101,7 @@ public class RestartIT extends ConfigurableMacIT {
     c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
"5K");
     Process ingest = cluster.exec(TestIngest.class, 
         "-u", "root", "-p", ROOT_PASSWORD, 
-        "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers());
+        "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "--rows", ""+VOPTS.rows);
     for (ProcessReference master : cluster.getProcesses().get(ServerType.MASTER)) {
       cluster.killProcess(ServerType.MASTER, master);
     }
@@ -108,12 +111,11 @@ public class RestartIT extends ConfigurableMacIT {
     ingest.destroy();
   }
   
-  @Test(timeout = 100 * 60 * 1000)
+  @Test(timeout = 10 * 60 * 1000)
   public void killedTabletServer() throws Exception {
     Connector c = getConnector();
     c.tableOperations().create("test_ingest");
-    TestIngest.Opts opts = new TestIngest.Opts();
-    TestIngest.ingest(c, opts, BWOPTS);
+    TestIngest.ingest(c, OPTS, BWOPTS);
     VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
     List<ProcessReference> procs = new ArrayList<ProcessReference>(cluster.getProcesses().get(ServerType.TABLET_SERVER));
     for (ProcessReference tserver : procs) {
@@ -142,8 +144,7 @@ public class RestartIT extends ConfigurableMacIT {
   public void killedTabletServerDuringShutdown() throws Exception {
     Connector c = getConnector();
     c.tableOperations().create("test_ingest");
-    TestIngest.Opts opts = new TestIngest.Opts();
-    TestIngest.ingest(c, opts, BWOPTS);
+    TestIngest.ingest(c, OPTS, BWOPTS);
     List<ProcessReference> procs = new ArrayList<ProcessReference>(cluster.getProcesses().get(ServerType.TABLET_SERVER));
     cluster.killProcess(ServerType.TABLET_SERVER, procs.get(0));
     assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index 28a0b19..31402ba 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@ -38,7 +38,7 @@ public class RestartStressIT extends ConfigurableMacIT {
     Map<String, String> opts = new HashMap<String, String>();
     opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
     opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
-    opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "500K");
+    opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
     opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
     opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
     cfg.setSiteConfig(opts);
@@ -50,7 +50,7 @@ public class RestartStressIT extends ConfigurableMacIT {
   static {
     IOPTS = new TestIngest.Opts();
     VOPTS = new VerifyIngest.Opts();
-    IOPTS.rows = VOPTS.rows = 100*1000;
+    IOPTS.rows = VOPTS.rows = 10*1000;
   }
   private static final ScannerOpts SOPTS = new ScannerOpts();
   

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index e976493..a55b543 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@ -259,7 +259,7 @@ public class SplitRecoveryIT extends ConfigurableMacIT {
     new SplitRecoveryIT().run();
   }
   
-  @Test(timeout = 10 * 1000)
+  @Test(timeout = 30 * 1000)
   public void test() throws Exception {
     assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/test/src/test/resources/log4j.properties b/test/src/test/resources/log4j.properties
index 277822a..27e4db2 100644
--- a/test/src/test/resources/log4j.properties
+++ b/test/src/test/resources/log4j.properties
@@ -18,6 +18,7 @@ log4j.appender.CA=org.apache.log4j.ConsoleAppender
 log4j.appender.CA.layout=org.apache.log4j.PatternLayout
 log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{2}] %-5p: %m%n
 
+log4j.logger.org.apache.accumulo.core=TRACE
 log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
 log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=off
 log4j.logger.org.apache.accumulo.core.util.shell.Shell=FATAL
@@ -27,6 +28,6 @@ log4j.logger.org.apache.hadoop.mapred=ERROR
 log4j.logger.org.apache.hadoop.tools.DistCp=WARN
 log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
 log4j.logger.org.apache.hadoop.util.ProcessTree=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
-log4j.logger.org.apache.zookeeper.ZooKeeper=WARN
+#log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+#log4j.logger.org.apache.zookeeper.ZooKeeper=WARN
 log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO


Mime
View raw message