cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmcken...@apache.org
Subject [1/3] git commit: Shutdown JVM on file descriptor exhaustion
Date Mon, 27 Oct 2014 21:46:20 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 7e7cb6e37 -> 95f966697


Shutdown JVM on file descriptor exhaustion

patch by jmckenzie, reviewed by marcuse for CASSANDRA-7579


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f25da979
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f25da979
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f25da979

Branch: refs/heads/trunk
Commit: f25da979f605d2484042e275284d55ce7d6f9142
Parents: 6e62030
Author: Joshua McKenzie <jmckenzie@apache.org>
Authored: Mon Oct 27 13:49:07 2014 -0500
Committer: Joshua McKenzie <jmckenzie@apache.org>
Committed: Mon Oct 27 13:49:07 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/cache/AutoSavingCache.java |  2 +
 .../org/apache/cassandra/cli/CliClient.java     |  4 +-
 src/java/org/apache/cassandra/cli/CliMain.java  |  2 +
 .../apache/cassandra/cli/CliSessionState.java   |  2 +
 .../cassandra/config/DatabaseDescriptor.java    |  2 +
 .../cassandra/db/BlacklistedDirectories.java    |  3 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  1 +
 .../cassandra/db/HintedHandOffManager.java      |  2 +
 .../compaction/AbstractCompactionStrategy.java  |  2 +
 src/java/org/apache/cassandra/gms/Gossiper.java |  1 +
 .../hadoop/ColumnFamilyRecordWriter.java        |  2 +
 .../apache/cassandra/hadoop/HadoopCompat.java   |  2 +
 .../cassandra/hadoop/cql3/CqlRecordWriter.java  |  6 +-
 .../cassandra/hadoop/pig/CassandraStorage.java  |  4 +-
 .../apache/cassandra/hadoop/pig/CqlStorage.java |  2 +
 .../cassandra/io/compress/SnappyCompressor.java |  3 +
 .../cassandra/io/util/MmappedSegmentedFile.java |  2 +
 .../cassandra/locator/CloudstackSnitch.java     |  2 +
 .../cassandra/net/OutboundTcpConnection.java    |  2 +
 .../apache/cassandra/service/ClientState.java   |  2 +
 .../cassandra/streaming/StreamLockfile.java     |  2 +
 .../cassandra/streaming/StreamSession.java      |  2 +
 .../thrift/CustomTThreadPoolServer.java         |  6 +-
 .../org/apache/cassandra/tools/BulkLoader.java  |  2 +
 .../org/apache/cassandra/tools/NodeTool.java    |  4 +-
 .../apache/cassandra/tools/SSTableImport.java   |  2 +
 .../cassandra/tools/StandaloneScrubber.java     |  2 +
 .../cassandra/tools/StandaloneSplitter.java     |  2 +
 .../cassandra/tools/StandaloneUpgrader.java     |  2 +
 .../org/apache/cassandra/transport/Client.java  |  2 +
 .../cassandra/transport/FrameCompressor.java    |  3 +
 .../transport/messages/BatchMessage.java        |  2 +
 .../transport/messages/ExecuteMessage.java      |  2 +
 .../transport/messages/PrepareMessage.java      |  2 +
 .../transport/messages/QueryMessage.java        |  2 +
 .../org/apache/cassandra/utils/CLibrary.java    |  1 +
 .../org/apache/cassandra/utils/FBUtilities.java |  1 +
 src/java/org/apache/cassandra/utils/Hex.java    |  1 +
 .../cassandra/utils/JVMStabilityInspector.java  |  8 ++
 .../org/apache/cassandra/db/CommitLogTest.java  |  1 -
 .../utils/JVMStabilityInspectorTest.java        | 36 ++++++++
 .../apache/cassandra/utils/KillerForTests.java  | 86 ++++++++++----------
 43 files changed, 169 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4e978a3..fbcf460 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.2
+ * Shutdown JVM on file descriptor exhaustion (CASSANDRA-7579)
  * Add 'die' policy for commit log and disk failure (CASSANDRA-7927)
  * Fix installing as service on Windows (CASSANDRA-8115)
  * Fix CREATE TABLE for CQL2 (CASSANDRA-8144)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/cache/AutoSavingCache.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
index d8fd5e0..1a861f8 100644
--- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java
+++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.io.util.LengthAvailableInputStream;
 import org.apache.cassandra.io.util.SequentialWriter;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 
 public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K, V>
@@ -142,6 +143,7 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 logger.debug(String.format("harmless error reading saved cache %s", path.getAbsolutePath()), e);
             }
             finally

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/cli/CliClient.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java
index c6b3bcd..45728c5 100644
--- a/src/java/org/apache/cassandra/cli/CliClient.java
+++ b/src/java/org/apache/cassandra/cli/CliClient.java
@@ -32,7 +32,6 @@ import com.google.common.base.Predicate;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterables;
 
-import org.apache.cassandra.serializers.MarshalException;
 import org.apache.commons.lang3.StringUtils;
 
 import org.antlr.runtime.tree.Tree;
@@ -44,10 +43,12 @@ import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.locator.SimpleSnitch;
+import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.tools.NodeProbe;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.UUIDGen;
 import org.apache.thrift.TBaseHelper;
 import org.apache.thrift.TException;
@@ -1658,6 +1659,7 @@ public class CliClient
             }
             catch (Exception ne)
             {
+                JVMStabilityInspector.inspectThrowable(ne);
                 String functions = Function.getFunctionNames();
                 sessionState.out.println("Type '" + defaultType + "' was not found. Available: " + functions
                                          + " Or any class which extends o.a.c.db.marshal.AbstractType.");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/cli/CliMain.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cli/CliMain.java b/src/java/org/apache/cassandra/cli/CliMain.java
index 7cfc77b..862b1af 100644
--- a/src/java/org/apache/cassandra/cli/CliMain.java
+++ b/src/java/org/apache/cassandra/cli/CliMain.java
@@ -28,6 +28,7 @@ import org.apache.cassandra.auth.IAuthenticator;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.transport.TTransport;
@@ -139,6 +140,7 @@ public class CliMain
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             sessionState.err.println("Exception retrieving information about the cassandra node, check you have connected to the thrift port.");
 
             e.printStackTrace(sessionState.err);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/cli/CliSessionState.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cli/CliSessionState.java b/src/java/org/apache/cassandra/cli/CliSessionState.java
index f0de713..2bfe7fc 100644
--- a/src/java/org/apache/cassandra/cli/CliSessionState.java
+++ b/src/java/org/apache/cassandra/cli/CliSessionState.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
 import org.apache.cassandra.thrift.ITransportFactory;
 import org.apache.cassandra.thrift.TFramedTransportFactory;
 import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * Used to hold the state for the CLI.
@@ -86,6 +87,7 @@ public class CliSessionState
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             err.printf("WARNING: Could not connect to the JMX on %s:%d - some information won't be shown.%n%n", hostName, jmxPort);
         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 34bd39a..ba84610 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -68,6 +68,7 @@ import org.apache.cassandra.scheduler.NoScheduler;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.memory.HeapPool;
 import org.apache.cassandra.utils.memory.NativePool;
 import org.apache.cassandra.utils.memory.MemtablePool;
@@ -142,6 +143,7 @@ public class DatabaseDescriptor
         {
             logger.error("Fatal error during configuration loading", e);
             System.err.println(e.getMessage() + "\nFatal error during configuration loading; unable to start. See log for stacktrace.");
+            JVMStabilityInspector.inspectThrowable(e);
             System.exit(1);
         }
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/db/BlacklistedDirectories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/BlacklistedDirectories.java b/src/java/org/apache/cassandra/db/BlacklistedDirectories.java
index de6c968..f47fd57 100644
--- a/src/java/org/apache/cassandra/db/BlacklistedDirectories.java
+++ b/src/java/org/apache/cassandra/db/BlacklistedDirectories.java
@@ -29,6 +29,8 @@ import java.util.concurrent.CopyOnWriteArraySet;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
 public class BlacklistedDirectories implements BlacklistedDirectoriesMBean
 {
     public static final String MBEAN_NAME = "org.apache.cassandra.db:type=BlacklistedDirectories";
@@ -48,6 +50,7 @@ public class BlacklistedDirectories implements BlacklistedDirectoriesMBean
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             logger.error("error registering MBean {}", MBEAN_NAME, e);
             //Allow the server to start even if the bean can't be registered
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index a45d1b2..184399a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -362,6 +362,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             // this shouldn't block anything.
             logger.warn("Failed unregistering mbean: {}", mbeanName, e);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
index 2eb49c7..593bebc 100644
--- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
@@ -65,6 +65,7 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.UUIDGen;
 import org.cliffc.high_scale_lib.NonBlockingHashSet;
 
@@ -222,6 +223,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     logger.warn("Could not delete hints for {}: {}", endpoint, e);
                 }
             }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 97696a8..288c475 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * Pluggable compaction strategy determines how SSTables get merged.
@@ -308,6 +309,7 @@ public abstract class AbstractCompactionStrategy
                 }
                 catch (Throwable t2)
                 {
+                    JVMStabilityInspector.inspectThrowable(t2);
                     if (t == null)
                         t = t2;
                     else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/gms/Gossiper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index 3fdee88..7e60881 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -178,6 +178,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 logger.error("Gossip error", e);
             }
             finally

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
index d6a873b..a1b04f0 100644
--- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
+++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
@@ -26,6 +26,7 @@ import java.util.*;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.thrift.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -198,6 +199,7 @@ final class ColumnFamilyRecordWriter extends AbstractColumnFamilyRecordWriter<By
                     }
                     catch (Exception e)
                     {
+                        JVMStabilityInspector.inspectThrowable(e);
                         closeInternal();
                         if (!iter.hasNext())
                         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/HadoopCompat.java b/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
index f2f7033..bfc4ff6 100644
--- a/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
+++ b/src/java/org/apache/cassandra/hadoop/HadoopCompat.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.StatusReporter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /*
  * This is based on ContextFactory.java from hadoop-2.0.x sources.
@@ -131,6 +132,7 @@ public class HadoopCompat {
                     get_counter = Class.forName(PACKAGE + ".TaskAttemptContext").getMethod("getCounter", String.class,
                             String.class);
                 } catch (Exception e) {
+                    JVMStabilityInspector.inspectThrowable(e);
                     get_counter = Class.forName(PACKAGE + ".TaskInputOutputContext").getMethod("getCounter",
                             String.class, String.class);
                 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
index 5075be4..ff1989d 100644
--- a/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
+++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlRecordWriter.java
@@ -23,8 +23,6 @@ import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.cassandra.hadoop.HadoopCompat;
-import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -38,11 +36,14 @@ import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.hadoop.AbstractColumnFamilyOutputFormat;
 import org.apache.cassandra.hadoop.AbstractColumnFamilyRecordWriter;
 import org.apache.cassandra.hadoop.ConfigHelper;
+import org.apache.cassandra.hadoop.HadoopCompat;
 import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.Progressable;
 import org.apache.thrift.TException;
 import org.apache.thrift.transport.TTransport;
 
@@ -250,6 +251,7 @@ class CqlRecordWriter extends AbstractColumnFamilyRecordWriter<Map<String, ByteB
                     }
                     catch (Exception e)
                     {
+                        JVMStabilityInspector.inspectThrowable(e);
                         closeInternal();
                         if (!iter.hasNext())
                         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
index 14d30d5..6b22fac 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.hadoop.mapreduce.*;
 import org.apache.pig.Expression;
 import org.apache.pig.ResourceSchema;
@@ -236,7 +237,8 @@ public class CassandraStorage extends AbstractCassandraStorage
                 }
                 catch (Exception e)
                 {
-                    cql3Table = true;                  
+                    JVMStabilityInspector.inspectThrowable(e);
+                    cql3Table = true;
                 }
                 if (hasColumn)
                 {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java b/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
index bb3ba36..7a6be71 100644
--- a/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
+++ b/src/java/org/apache/cassandra/hadoop/pig/CqlStorage.java
@@ -34,6 +34,7 @@ import org.apache.cassandra.hadoop.cql3.CqlConfigHelper;
 import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.hadoop.mapreduce.*;
 import org.apache.pig.Expression;
 import org.apache.pig.Expression.OpType;
@@ -602,6 +603,7 @@ public class CqlStorage extends AbstractCassandraStorage
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     // no compact column at value_alias
                 }
             }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/io/compress/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/SnappyCompressor.java b/src/java/org/apache/cassandra/io/compress/SnappyCompressor.java
index 7ceaa95..3583201 100644
--- a/src/java/org/apache/cassandra/io/compress/SnappyCompressor.java
+++ b/src/java/org/apache/cassandra/io/compress/SnappyCompressor.java
@@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory;
 import org.xerial.snappy.Snappy;
 import org.xerial.snappy.SnappyError;
 
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
 public class SnappyCompressor implements ICompressor
 {
     public static final SnappyCompressor instance = new SnappyCompressor();
@@ -57,6 +59,7 @@ public class SnappyCompressor implements ICompressor
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             return false;
         }
         catch (NoClassDefFoundError e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
index 450553b..ccc03fc 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
@@ -28,6 +28,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 public class MmappedSegmentedFile extends SegmentedFile
 {
@@ -105,6 +106,7 @@ public class MmappedSegmentedFile extends SegmentedFile
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             // This is not supposed to happen
             logger.error("Error while unmapping segments", e);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
index 0200f0f..7f72ae2 100644
--- a/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
+++ b/src/java/org/apache/cassandra/locator/CloudstackSnitch.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * A snitch that assumes a Cloudstack Zone follows the typical convention
@@ -160,6 +161,7 @@ public class CloudstackSnitch extends AbstractNetworkTopologySnitch
             } 
             catch (Exception e) 
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 continue;
             }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java b/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
index e1cb7a2..996e606 100644
--- a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
+++ b/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
@@ -48,6 +48,7 @@ import org.apache.cassandra.io.util.DataOutputStreamPlus;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.UUIDGen;
 import org.xerial.snappy.SnappyOutputStream;
 
@@ -168,6 +169,7 @@ public class OutboundTcpConnection extends Thread
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     // really shouldn't get here, as exception handling in writeConnected() is reasonably robust
                     // but we want to catch anything bad we don't drop the messages in the current batch
                     logger.error("error processing a message intended for {}", poolReference.endPoint(), e);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/service/ClientState.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java
index 492a07a..b4b162c 100644
--- a/src/java/org/apache/cassandra/service/ClientState.java
+++ b/src/java/org/apache/cassandra/service/ClientState.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.thrift.ThriftValidation;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.SemanticVersion;
 
@@ -84,6 +85,7 @@ public class ClientState
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 logger.info("Cannot use class {} as query handler ({}), ignoring by defaulting on normal query handling", customHandlerClass, e.getMessage());
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/streaming/StreamLockfile.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamLockfile.java b/src/java/org/apache/cassandra/streaming/StreamLockfile.java
index 4d20479..0b1d832 100644
--- a/src/java/org/apache/cassandra/streaming/StreamLockfile.java
+++ b/src/java/org/apache/cassandra/streaming/StreamLockfile.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableWriter;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * Encapsulates the behavior for 'locking' any streamed sttables to a node.
@@ -105,6 +106,7 @@ public class StreamLockfile
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 logger.warn("failed to delete a potentially stale sstable {}", file);
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 3ba296c..b03d28a 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.metrics.StreamingMetrics;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.messages.*;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 
 /**
@@ -217,6 +218,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             onError(e);
         }
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java b/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
index f7a602c..59246df 100644
--- a/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
+++ b/src/java/org/apache/cassandra/thrift/CustomTThreadPoolServer.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.concurrent.NamedThreadFactory;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.thrift.TException;
 import org.apache.thrift.TProcessor;
 import org.apache.thrift.protocol.TProtocol;
@@ -214,9 +215,10 @@ public class CustomTThreadPoolServer extends TServer
             {
                 logger.error("Thrift error occurred during processing of message.", tx);
             }
-            catch (Exception x)
+            catch (Exception e)
             {
-                logger.error("Error occurred during processing of message.", x);
+                JVMStabilityInspector.inspectThrowable(e);
+                logger.error("Error occurred during processing of message.", e);
             }
             finally
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/BulkLoader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/BulkLoader.java b/src/java/org/apache/cassandra/tools/BulkLoader.java
index 50e340b..84870a0 100644
--- a/src/java/org/apache/cassandra/tools/BulkLoader.java
+++ b/src/java/org/apache/cassandra/tools/BulkLoader.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.io.sstable.SSTableLoader;
 import org.apache.cassandra.streaming.*;
 import org.apache.cassandra.thrift.*;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 
 public class BulkLoader
@@ -107,6 +108,7 @@ public class BulkLoader
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             System.err.println(e.getMessage());
             if (e.getCause() != null)
                 System.err.println(e.getCause());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index 7b332f2..d5180f0 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -54,6 +54,7 @@ import org.apache.cassandra.streaming.SessionInfo;
 import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.utils.EstimatedHistogram;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
@@ -2170,8 +2171,9 @@ public class NodeTool
             try
             {
                 probe.stopCassandraDaemon();
-            } catch (Exception ignored)
+            } catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 // ignored
             }
             System.out.println("Cassandra has shutdown.");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 75ad26f..edad02a 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -46,6 +46,7 @@ import org.apache.cassandra.io.sstable.SSTableWriter;
 import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonParser;
 import org.codehaus.jackson.JsonToken;
@@ -514,6 +515,7 @@ public class SSTableImport
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             e.printStackTrace();
             System.err.println("ERROR: " + e.getMessage());
             System.exit(-1);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 78d4d9e..42799a5 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -32,6 +32,7 @@ import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.LeveledManifest;
 import org.apache.cassandra.db.compaction.Scrubber;
 import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 
 import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -86,6 +87,7 @@ public class StandaloneScrubber
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
                     if (options.debug)
                         e.printStackTrace(System.err);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index 75ad959..9d1fbf8 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -30,6 +30,7 @@ import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.SSTableSplitter;
 import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 
 import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -131,6 +132,7 @@ public class StandaloneSplitter
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
                     if (options.debug)
                         e.printStackTrace(System.err);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 55f206e..f632dfd 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -29,6 +29,7 @@ import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.Upgrader;
 import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 
 import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
@@ -80,6 +81,7 @@ public class StandaloneUpgrader
                 }
                 catch (Exception e)
                 {
+                    JVMStabilityInspector.inspectThrowable(e);
                     System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
                     if (options.debug)
                         e.printStackTrace(System.err);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/Client.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/Client.java b/src/java/org/apache/cassandra/transport/Client.java
index 989b954..c7c3103 100644
--- a/src/java/org/apache/cassandra/transport/Client.java
+++ b/src/java/org/apache/cassandra/transport/Client.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.transport.messages.*;
 import org.apache.cassandra.utils.Hex;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
 
 import static org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
@@ -80,6 +81,7 @@ public class Client extends SimpleClient
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 System.err.println("ERROR: " + e.getMessage());
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/FrameCompressor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/FrameCompressor.java b/src/java/org/apache/cassandra/transport/FrameCompressor.java
index 8ab735f..fa337ec 100644
--- a/src/java/org/apache/cassandra/transport/FrameCompressor.java
+++ b/src/java/org/apache/cassandra/transport/FrameCompressor.java
@@ -27,6 +27,8 @@ import org.xerial.snappy.SnappyError;
 import net.jpountz.lz4.LZ4Exception;
 import net.jpountz.lz4.LZ4Factory;
 
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
 public interface FrameCompressor
 {
     public Frame compress(Frame frame) throws IOException;
@@ -48,6 +50,7 @@ public interface FrameCompressor
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 i = null;
             }
             catch (NoClassDefFoundError e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
index 19fa6aa..ff791ef 100644
--- a/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/BatchMessage.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
 import org.apache.cassandra.utils.UUIDGen;
 
@@ -220,6 +221,7 @@ public class BatchMessage extends Message.Request
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             return ErrorMessage.fromException(e);
         }
         finally

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
index 78182b2..324ae00 100644
--- a/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java
@@ -33,6 +33,7 @@ import org.apache.cassandra.exceptions.PreparedQueryNotFoundException;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MD5Digest;
 import org.apache.cassandra.utils.UUIDGen;
 
@@ -141,6 +142,7 @@ public class ExecuteMessage extends Message.Request
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             return ErrorMessage.fromException(e);
         }
         finally

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
index 2899cf8..221b9ca 100644
--- a/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/PrepareMessage.java
@@ -26,6 +26,7 @@ import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.UUIDGen;
 
 public class PrepareMessage extends Message.Request
@@ -83,6 +84,7 @@ public class PrepareMessage extends Message.Request
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             return ErrorMessage.fromException(e);
         }
         finally

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
index 377cdaf..ff50422 100644
--- a/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
+++ b/src/java/org/apache/cassandra/transport/messages/QueryMessage.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.exceptions.*;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.*;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.UUIDGen;
 
 /**
@@ -126,6 +127,7 @@ public class QueryMessage extends Message.Request
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             if (!((e instanceof RequestValidationException) || (e instanceof RequestExecutionException)))
                 logger.error("Unexpected error during query", e);
             return ErrorMessage.fromException(e);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/utils/CLibrary.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/CLibrary.java b/src/java/org/apache/cassandra/utils/CLibrary.java
index 1927578..1c771af 100644
--- a/src/java/org/apache/cassandra/utils/CLibrary.java
+++ b/src/java/org/apache/cassandra/utils/CLibrary.java
@@ -296,6 +296,7 @@ public final class CLibrary
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             logger.warn("unable to read fd field from FileDescriptor");
         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/utils/FBUtilities.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java
index 7e9adab..01b7923 100644
--- a/src/java/org/apache/cassandra/utils/FBUtilities.java
+++ b/src/java/org/apache/cassandra/utils/FBUtilities.java
@@ -364,6 +364,7 @@ public class FBUtilities
         }
         catch (Exception e)
         {
+            JVMStabilityInspector.inspectThrowable(e);
             logger.warn("Unable to load version.properties", e);
             return "debug version";
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/utils/Hex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/Hex.java b/src/java/org/apache/cassandra/utils/Hex.java
index 5ba0b1a..f018f26 100644
--- a/src/java/org/apache/cassandra/utils/Hex.java
+++ b/src/java/org/apache/cassandra/utils/Hex.java
@@ -105,6 +105,7 @@ public class Hex
             }
             catch (Exception e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 // Swallowing as we'll just use a copying constructor
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index bcff172..2883ab3 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.utils;
 
+import java.io.FileNotFoundException;
+import java.net.SocketException;
+
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,6 +53,11 @@ public final class JVMStabilityInspector
 
         if (DatabaseDescriptor.getDiskFailurePolicy() == Config.DiskFailurePolicy.die)
             if (t instanceof FSError || t instanceof CorruptSSTableException)
+            isUnstable = true;
+
+        // Check for file handle exhaustion
+        if (t instanceof FileNotFoundException || t instanceof SocketException)
+            if (t.getMessage().contains("Too many open files"))
                 isUnstable = true;
 
         if (isUnstable)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 8a1bb0c..9e59797 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -302,7 +302,6 @@ public class CommitLogTest extends SchemaLoader
         KillerForTests killerForTests = new KillerForTests();
         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
-
         try
         {
             DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
index e2a5107..494b3b1 100644
--- a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
+++ b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java
@@ -5,7 +5,14 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.FSReadError;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -48,4 +55,33 @@ public class JVMStabilityInspectorTest
         }
     }
 
+    @Test
+    public void fileHandleTest() throws FileNotFoundException
+    {
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+
+        try
+        {
+            killerForTests.reset();
+            JVMStabilityInspector.inspectThrowable(new SocketException("Should not fail"));
+            assertFalse(killerForTests.wasKilled());
+
+            killerForTests.reset();
+            JVMStabilityInspector.inspectThrowable(new FileNotFoundException("Also should not fail"));
+            assertFalse(killerForTests.wasKilled());
+
+            killerForTests.reset();
+            JVMStabilityInspector.inspectThrowable(new SocketException("Too many open files"));
+            assertTrue(killerForTests.wasKilled());
+
+            killerForTests.reset();
+            JVMStabilityInspector.inspectCommitLogThrowable(new FileNotFoundException("Too many open files"));
+            assertTrue(killerForTests.wasKilled());
+        }
+        finally
+        {
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f25da979/test/unit/org/apache/cassandra/utils/KillerForTests.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index 83cd7fc..d488f97 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -1,43 +1,43 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.utils;
-
-/**
- * Responsible for stubbing out the System.exit() logic during unit tests.
- */
-public class KillerForTests extends JVMStabilityInspector.Killer
-{
-    private boolean killed = false;
-
-    @Override
-    protected void killCurrentJVM(Throwable t)
-    {
-        killed = true;
-    }
-
-    public boolean wasKilled()
-    {
-        return killed;
-    }
-
-    public void reset()
-    {
-        killed = false;
-    }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+/**
+ * Responsible for stubbing out the System.exit() logic during unit tests.
+ */
+public class KillerForTests extends JVMStabilityInspector.Killer
+{
+    private boolean killed = false;
+
+    @Override
+    protected void killCurrentJVM(Throwable t)
+    {
+        killed = true;
+    }
+
+    public boolean wasKilled()
+    {
+        return killed;
+    }
+
+    public void reset()
+    {
+        killed = false;
+    }
+}


Mime
View raw message