activemq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From clebertsuco...@apache.org
Subject [1/2] activemq-artemis git commit: ARTEMIS-1489 CriticalAnalyzer Fault Injection Test
Date Wed, 15 Nov 2017 18:39:26 GMT
Repository: activemq-artemis
Updated Branches:
  refs/heads/master 32802eeba -> a9b02fb68


ARTEMIS-1489 CriticalAnalyzer Fault Injection Test

Fault injection via Byteman to simulate deadlock and slow disk
sync scenarios, ensures the Critical Analyzer detects the issues
reacts appropriately.


Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/2e7b660c
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/2e7b660c
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/2e7b660c

Branch: refs/heads/master
Commit: 2e7b660ca80aa1fbb476dfa1282ead1d21ab4b98
Parents: 32802ee
Author: Martyn Taylor <mtaylor@redhat.com>
Authored: Tue Oct 31 10:10:30 2017 +0000
Committer: Clebert Suconic <clebertsuconic@apache.org>
Committed: Wed Nov 15 13:37:29 2017 -0500

----------------------------------------------------------------------
 .../CriticalAnalyzerFaultInjectionTestBase.java | 91 ++++++++++++++++++++
 .../analyzer/DeadlockStorageManagerTest.java    | 81 +++++++++++++++++
 .../critical/analyzer/DeadlockedQueueTest.java  | 48 +++++++++++
 .../analyzer/FileSystemSyncBlockedTest.java     | 49 +++++++++++
 4 files changed, 269 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/2e7b660c/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/CriticalAnalyzerFaultInjectionTestBase.java
----------------------------------------------------------------------
diff --git a/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/CriticalAnalyzerFaultInjectionTestBase.java
b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/CriticalAnalyzerFaultInjectionTestBase.java
new file mode 100644
index 0000000..156edd7
--- /dev/null
+++ b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/CriticalAnalyzerFaultInjectionTestBase.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
+
+import javax.jms.DeliveryMode;
+import javax.jms.MessageProducer;
+import javax.jms.Queue;
+import javax.jms.Session;
+
+import org.apache.activemq.artemis.api.core.RoutingType;
+import org.apache.activemq.artemis.api.core.SimpleString;
+import org.apache.activemq.artemis.core.config.Configuration;
+import org.apache.activemq.artemis.core.server.JournalType;
+import org.apache.activemq.artemis.core.server.impl.AddressInfo;
+import org.apache.activemq.artemis.tests.util.JMSTestBase;
+import org.apache.activemq.artemis.tests.util.Wait;
+import org.apache.activemq.artemis.utils.critical.CriticalAnalyzerPolicy;
+import org.junit.Before;
+
+public abstract class CriticalAnalyzerFaultInjectionTestBase extends JMSTestBase {
+
+   // Critical Analyzer Settings
+   private static long CHECK_PERIOD = 100;
+   private static long TIMEOUT = 3000;
+
+   private SimpleString address = SimpleString.toSimpleString("faultInjectionTestAddress");
+
+   private Thread t;
+
+   @Override
+   @Before
+   public void setUp() throws Exception {
+      super.setUp();
+      server.addAddressInfo(new AddressInfo(address, RoutingType.ANYCAST));
+      server.createQueue(address, RoutingType.ANYCAST, address, null, true, false);
+      conn = nettyCf.createConnection();
+   }
+
+   /*
+   Checks every 100ms timesout after 3000ms.  Test should wait no longer than 3100s + Shutdown
time.
+    */
+   @Override
+   protected Configuration createDefaultConfig(boolean netty) throws Exception {
+      return super.createDefaultConfig(netty).setCriticalAnalyzerPolicy(CriticalAnalyzerPolicy.SHUTDOWN).setCriticalAnalyzer(true).setCriticalAnalyzerCheckPeriod(CHECK_PERIOD).setCriticalAnalyzerTimeout(TIMEOUT).setJournalType(JournalType.NIO);
+   }
+
+   @Override
+   public boolean usePersistence() {
+      return true;
+   }
+
+   void testSendDurableMessage() throws Exception {
+      try {
+         t = new Thread(() -> {
+            try {
+               Session s = conn.createSession(true, Session.SESSION_TRANSACTED);
+
+               Queue jmsQueue = s.createQueue(address.toString());
+               MessageProducer p = s.createProducer(jmsQueue);
+               p.setDeliveryMode(DeliveryMode.PERSISTENT);
+               conn.start();
+               for (int i = 0; i < 10; i++) {
+                  p.send(s.createTextMessage("payload"));
+               }
+               s.commit();
+            } catch (Throwable e) {
+               e.printStackTrace();
+            }
+         });
+         t.start();
+
+         Wait.assertFalse(server::isStarted);
+      } finally {
+         t.interrupt();
+      }
+   }
+}

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/2e7b660c/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockStorageManagerTest.java
----------------------------------------------------------------------
diff --git a/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockStorageManagerTest.java
b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockStorageManagerTest.java
new file mode 100644
index 0000000..34d07a5
--- /dev/null
+++ b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockStorageManagerTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.activemq.artemis.core.persistence.impl.journal.AbstractJournalStorageManager;
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(BMUnitRunner.class)
+public class DeadlockStorageManagerTest extends CriticalAnalyzerFaultInjectionTestBase {
+
+   private static Thread lockT;
+
+   @BMRules(
+      rules = {
+         @BMRule(
+            name = "Deadlock on Storage Manager",
+            targetClass = "org.apache.activemq.artemis.core.persistence.impl.journal.AbstractJournalStorageManager",
+            targetMethod = "commit",
+            targetLocation = "ENTRY",
+            condition = "!flagged(\"testDeadlockOnStorageManager\")",
+            action = "org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer.DeadlockStorageManagerTest.acquireStorageManagerLock($0);"),
+         @BMRule(
+            name = "Release Suspended Thread during Server Shutdown", // Releases wakes up
suspended threads to allow shutdown to complete
+            targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
+            targetMethod = "stop",
+            targetLocation = "ENTRY",
+            action = "flag(\"testDeadlockOnStorageManager\"); org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer.DeadlockStorageManagerTest.releaseStorageManagerLock()"
)
+      })
+   @Test(timeout = 60000)
+   public void testDeadlockOnStorageManager() throws Exception {
+      testSendDurableMessage();
+   }
+
+   public static void acquireStorageManagerLock(final Object o) throws Exception {
+      // We're attempting to cause a deadlock on the readlock.  This means we need to grab
the write lock
+      // in a separate thread.
+      if (lockT == null) {
+         lockT = new Thread(() -> {
+            try {
+               Field field = AbstractJournalStorageManager.class.getDeclaredField("storageManagerLock");
+               field.setAccessible(true);
+               ReentrantReadWriteLock lock = ((ReentrantReadWriteLock) field.get(o));
+               lock.writeLock().lock();
+               try {
+                  Thread.sleep(10000);
+               } catch (InterruptedException ie) {
+                  lock.writeLock().unlock();
+               }
+            } catch (Exception e) {
+               fail();
+            }
+         });
+         lockT.start();
+      }
+   }
+
+   public static void releaseStorageManagerLock() throws NoSuchFieldException, IllegalAccessException
{
+      lockT.interrupt();
+   }
+}

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/2e7b660c/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockedQueueTest.java
----------------------------------------------------------------------
diff --git a/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockedQueueTest.java
b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockedQueueTest.java
new file mode 100644
index 0000000..2c7ee13
--- /dev/null
+++ b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockedQueueTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
+
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(BMUnitRunner.class)
+public class DeadlockedQueueTest extends CriticalAnalyzerFaultInjectionTestBase {
+
+   @BMRules(
+      rules = {
+         @BMRule(
+            name = "Deadlock during Queue Delivery",
+            targetClass = "org.apache.activemq.artemis.core.server.impl.QueueImpl",
+            targetMethod = "deliver",
+            targetLocation = "AFTER SYNCHRONIZE",
+            condition = "!flagged(\"testDeadlockOnQueue\")",
+            action = "waitFor(\"testDeadlockOnQueue\")"),
+         @BMRule(
+            name = "Release Suspended Thread during Server Shutdown", // Releases wakes up
suspended threads to allow shutdown to complete
+            targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
+            targetMethod = "stop",
+            targetLocation = "ENTRY",
+            action = "flag(\"testDeadlockOnQueue\"); signalWake(\"testDeadlockOnQueue\")")
+      })
+   @Test(timeout = 60000)
+   public void testDeadlockOnQueue() throws Exception {
+      testSendDurableMessage();
+   }
+}

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/2e7b660c/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/FileSystemSyncBlockedTest.java
----------------------------------------------------------------------
diff --git a/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/FileSystemSyncBlockedTest.java
b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/FileSystemSyncBlockedTest.java
new file mode 100644
index 0000000..b14b3e6
--- /dev/null
+++ b/tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/FileSystemSyncBlockedTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
+
+import org.jboss.byteman.contrib.bmunit.BMRule;
+import org.jboss.byteman.contrib.bmunit.BMRules;
+import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(BMUnitRunner.class)
+public class FileSystemSyncBlockedTest extends CriticalAnalyzerFaultInjectionTestBase {
+
+   @BMRules(
+      rules = {
+         @BMRule(
+            name = "Simulate Slow Disk Sync",
+            targetClass = "org.apache.activemq.artemis.core.io.nio.NIOSequentialFile",
+            targetMethod = "sync",
+            targetLocation = "ENTRY",
+            condition = "!flagged(\"testSlowDiskSync\")",  // Once the server shutdowns we
stop applying this rule.
+            action = "waitFor(\"testSlowDiskSync\")"),
+         @BMRule(
+            // We ensure that no more
+            name = "Release Suspended Thread during Server Shutdown", // Releases wakes up
suspended threads to allow shutdown to complete
+            targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
+            targetMethod = "stop",
+            targetLocation = "ENTRY",
+            action = "flag(\"testSlowDiskSync\"); signalWake(\"testSlowDiskSync\")")
+      })
+   @Test(timeout = 60000)
+   public void testSlowDiskSync()  throws Exception {
+      testSendDurableMessage();
+   }
+}


Mime
View raw message