cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmcken...@apache.org
Subject git commit: Shutdown JVM on OOM
Date Thu, 09 Oct 2014 15:46:56 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.1 5a33a9ad2 -> 145213470


Shutdown JVM on OOM

patch by Josh McKenzie; reviewed by Aleksey Yeschenko for CASSANDRA-7507


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/14521347
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/14521347
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/14521347

Branch: refs/heads/cassandra-2.1
Commit: 1452134707e2a143def3c03d1db416b1c8197565
Parents: 5a33a9a
Author: Joshua McKenzie <josh.mckenzie@datastax.com>
Authored: Thu Oct 9 10:40:57 2014 -0500
Committer: Joshua McKenzie <josh.mckenzie@datastax.com>
Committed: Thu Oct 9 10:40:57 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/auth/CassandraAuthorizer.java     |  6 +--
 .../AbstractTracingAwareExecutorService.java    |  2 +
 .../DebuggableScheduledThreadPoolExecutor.java  |  7 ++-
 .../apache/cassandra/concurrent/SEPWorker.java  |  3 ++
 .../db/commitlog/CommitLogReplayer.java         |  1 +
 .../db/commitlog/CommitLogSegmentManager.java   |  2 +
 .../db/compaction/CompactionManager.java        |  7 +--
 .../cassandra/db/compaction/Scrubber.java       |  2 +
 src/java/org/apache/cassandra/gms/Gossiper.java |  2 +
 .../io/sstable/SSTableSimpleUnsortedWriter.java |  2 +
 .../org/apache/cassandra/io/util/FileUtils.java |  5 ++-
 .../cassandra/service/CassandraDaemon.java      | 20 ++-------
 .../cassandra/service/StorageService.java       | 18 ++++++--
 .../cassandra/streaming/ConnectionHandler.java  |  6 ++-
 .../streaming/messages/IncomingFileMessage.java |  6 ++-
 .../org/apache/cassandra/tools/NodeProbe.java   |  6 ++-
 .../cassandra/tools/SSTableLevelResetter.java   |  6 ++-
 .../org/apache/cassandra/transport/Message.java |  6 ++-
 .../org/apache/cassandra/utils/CLibrary.java    |  1 +
 .../cassandra/utils/FastByteOperations.java     |  4 +-
 .../cassandra/utils/JVMStabilityInspector.java  | 45 ++++++++++++++++++++
 .../apache/cassandra/utils/ResourceWatcher.java |  1 +
 23 files changed, 117 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 054870b..8d64d34 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.1
+ * Shutdown JVM on OOM (CASSANDRA-7507)
  * Upgrade netty version and enable epoll event loop (CASSANDRA-7761)
  * Don't duplicate sstables smaller than split size when using
    the sstablesplitter tool (CASSANDRA-7616)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
index 9b22e22..20060c0 100644
--- a/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
+++ b/src/java/org/apache/cassandra/auth/CassandraAuthorizer.java
@@ -185,7 +185,7 @@ public class CassandraAuthorizer implements IAuthorizer
         {
             process(String.format("DELETE FROM %s.%s WHERE username = '%s'", Auth.AUTH_KS,
PERMISSIONS_CF, escape(droppedUser)));
         }
-        catch (Throwable e)
+        catch (RequestExecutionException e)
         {
             logger.warn("CassandraAuthorizer failed to revoke all permissions of {}: {}",
droppedUser, e);
         }
@@ -204,7 +204,7 @@ public class CassandraAuthorizer implements IAuthorizer
                                          PERMISSIONS_CF,
                                          escape(droppedResource.getName())));
         }
-        catch (Throwable e)
+        catch (RequestExecutionException e)
         {
             logger.warn("CassandraAuthorizer failed to revoke all permissions on {}: {}",
droppedResource, e);
             return;
@@ -220,7 +220,7 @@ public class CassandraAuthorizer implements IAuthorizer
                                       escape(row.getString(USERNAME)),
                                       escape(droppedResource.getName())));
             }
-            catch (Throwable e)
+            catch (RequestExecutionException e)
             {
                 logger.warn("CassandraAuthorizer failed to revoke all permissions on {}:
{}", droppedResource, e);
             }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/concurrent/AbstractTracingAwareExecutorService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/concurrent/AbstractTracingAwareExecutorService.java
b/src/java/org/apache/cassandra/concurrent/AbstractTracingAwareExecutorService.java
index 8fa3ec9..fb753b0 100644
--- a/src/java/org/apache/cassandra/concurrent/AbstractTracingAwareExecutorService.java
+++ b/src/java/org/apache/cassandra/concurrent/AbstractTracingAwareExecutorService.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static org.apache.cassandra.tracing.Tracing.isTracing;
 
@@ -164,6 +165,7 @@ public abstract class AbstractTracingAwareExecutorService implements TracingAwar
             }
             catch (Throwable t)
             {
+                JVMStabilityInspector.inspectThrowable(t);
                 logger.warn("Uncaught exception on thread {}: {}", Thread.currentThread(),
t);
                 result = t;
                 failure = true;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
b/src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
index a41df54..4fc1d6c 100644
--- a/src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.concurrent;
 
 import java.util.concurrent.*;
 
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
 /**
  * Like DebuggableThreadPoolExecutor, DebuggableScheduledThreadPoolExecutor always
  * logs exceptions from the tasks it is given, even if Future.get is never called elsewhere.
@@ -74,9 +76,10 @@ public class DebuggableScheduledThreadPoolExecutor extends ScheduledThreadPoolEx
             {
                 runnable.run();
             }
-            catch (Throwable e)
+            catch (Throwable t)
             {
-                DebuggableThreadPoolExecutor.handleOrLog(e);
+                JVMStabilityInspector.inspectThrowable(t);
+                DebuggableThreadPoolExecutor.handleOrLog(t);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/concurrent/SEPWorker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/concurrent/SEPWorker.java b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
index 60a8d61..3b3e7ad 100644
--- a/src/java/org/apache/cassandra/concurrent/SEPWorker.java
+++ b/src/java/org/apache/cassandra/concurrent/SEPWorker.java
@@ -24,6 +24,8 @@ import java.util.concurrent.locks.LockSupport;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.utils.JVMStabilityInspector;
+
 final class SEPWorker extends AtomicReference<SEPWorker.Work> implements Runnable
 {
     private static final Logger logger = LoggerFactory.getLogger(SEPWorker.class);
@@ -122,6 +124,7 @@ final class SEPWorker extends AtomicReference<SEPWorker.Work> implements
Runnabl
         }
         catch (Throwable t)
         {
+            JVMStabilityInspector.inspectThrowable(t);
             while (true)
             {
                 if (get().assigned != null)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 1012829..e89338a 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -358,6 +358,7 @@ public class CommitLogReplayer
                     }
                     catch (Throwable t)
                     {
+                        JVMStabilityInspector.inspectThrowable(t);
                         File f = File.createTempFile("mutation", "dat");
                         DataOutputStream out = new DataOutputStream(new FileOutputStream(f));
                         try

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 05c4b9d..0771b7a 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -49,6 +49,7 @@ import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.Allocation;
@@ -159,6 +160,7 @@ public class CommitLogSegmentManager
                     }
                     catch (Throwable t)
                     {
+                        JVMStabilityInspector.inspectThrowable(t);
                         if (!CommitLog.handleCommitError("Failed managing commit log segments",
t))
                             return;
                         // sleep some arbitrary period to avoid spamming CL

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index e309cfb..51f45b8 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -35,7 +35,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
@@ -85,11 +84,8 @@ import org.apache.cassandra.metrics.CompactionMetrics;
 import org.apache.cassandra.repair.Validator;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.CloseableIterator;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.MerkleTree;
-import org.apache.cassandra.utils.WrappedRunnable;
 import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.*;
 
 /**
  * A singleton which manages a private executor of ongoing compactions.
@@ -1036,6 +1032,7 @@ public class CompactionManager implements CompactionManagerMBean
             }
             catch (Throwable e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 logger.error("Error anticompacting " + sstable, e);
                 repairedSSTableWriter.abort();
                 unRepairedSSTableWriter.abort();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 7303da1..b3d098d 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -29,6 +29,7 @@ import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 
 public class Scrubber implements Closeable
@@ -150,6 +151,7 @@ public class Scrubber implements Closeable
                 }
                 catch (Throwable th)
                 {
+                    JVMStabilityInspector.inspectThrowable(th);
                     outputHandler.warn("Error reading index file", th);
                     nextIndexKey = null;
                     nextRowPositionFromIndex = dataFile.length();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/gms/Gossiper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index 24b4409..bedc8d7 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -47,6 +47,7 @@ import org.apache.cassandra.net.MessageOut;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import com.google.common.collect.ImmutableList;
 
@@ -511,6 +512,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
             }
             catch (Throwable th)
             {
+                JVMStabilityInspector.inspectThrowable(th);
                 // TODO this is broken
                 logger.warn("Unable to calculate tokens for {}.  Will use a random one",
address);
                 tokens = Collections.singletonList(StorageService.getPartitioner().getRandomToken());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index ad3c451..3cfdc7b 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -36,6 +36,7 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.compress.CompressionParameters;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * A SSTable writer that doesn't assume rows are in sorted order.
@@ -217,6 +218,7 @@ public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
             }
             catch (Throwable e)
             {
+                JVMStabilityInspector.inspectThrowable(e);
                 if (writer != null)
                     writer.abort();
                 exception = e;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 41b7aa3..e590918 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -33,6 +33,8 @@ import java.nio.file.StandardCopyOption;
 import java.text.DecimalFormat;
 import java.util.Arrays;
 
+import sun.nio.ch.DirectBuffer;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,7 +47,7 @@ import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.service.StorageService;
-import sun.nio.ch.DirectBuffer;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 public class FileUtils
 {
@@ -69,6 +71,7 @@ public class FileUtils
         }
         catch (Throwable t)
         {
+            JVMStabilityInspector.inspectThrowable(t);
             logger.info("Cannot initialize un-mmaper.  (Are you using a non-Oracle JVM?)
 Compacted data files will not be removed promptly.  Consider using an Oracle JVM or using
standard disk access mode");
         }
         canCleanDirectBuffers = canClean;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index a0dacee..e68dc26 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -56,10 +56,7 @@ import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.thrift.ThriftServer;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.CLibrary;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Mx4jTool;
-import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.*;
 
 /**
  * The <code>CassandraDaemon</code> is an abstraction for a Cassandra daemon
@@ -71,16 +68,6 @@ public class CassandraDaemon
 {
     public static final String MBEAN_NAME = "org.apache.cassandra.db:type=NativeAccess";
 
-    // Have a dedicated thread to call exit to avoid deadlock in the case where the thread
that wants to invoke exit
-    // belongs to an executor that our shutdown hook wants to wait to exit gracefully. See
CASSANDRA-5273.
-    private static final Thread exitThread = new Thread(new Runnable()
-    {
-        public void run()
-        {
-            System.exit(100);
-        }
-    }, "Exit invoker");
-
     private static final Logger logger = LoggerFactory.getLogger(CassandraDaemon.class);
 
     private static final CassandraDaemon instance = new CassandraDaemon();
@@ -167,9 +154,7 @@ public class CassandraDaemon
                 Tracing.trace("Exception in thread {}", t, e);
                 for (Throwable e2 = e; e2 != null; e2 = e2.getCause())
                 {
-                    // some code, like FileChannel.map, will wrap an OutOfMemoryError in
another exception
-                    if (e2 instanceof OutOfMemoryError)
-                        exitThread.start();
+                    JVMStabilityInspector.inspectThrowable(e2);
 
                     if (e2 instanceof FSError)
                     {
@@ -287,6 +272,7 @@ public class CassandraDaemon
         }
         catch (Throwable t)
         {
+            JVMStabilityInspector.inspectThrowable(t);
             logger.warn("Unable to start GCInspector (currently only supported on the Sun
JVM)");
         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index d4bfe83..de8f576 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -144,6 +144,8 @@ public class StorageService extends NotificationBroadcasterSupport implements
IE
 
     public volatile VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(getPartitioner());
 
+    private Thread drainOnShutdown = null;
+
     public static final StorageService instance = new StorageService();
 
     public static IPartitioner getPartitioner()
@@ -577,7 +579,7 @@ public class StorageService extends NotificationBroadcasterSupport implements
IE
         }
 
         // daemon threads, like our executors', continue to run while shutdown hooks are
invoked
-        Thread drainOnShutdown = new Thread(new WrappedRunnable()
+        drainOnShutdown = new Thread(new WrappedRunnable()
         {
             @Override
             public void runMayThrow() throws InterruptedException
@@ -615,10 +617,11 @@ public class StorageService extends NotificationBroadcasterSupport implements
IE
                 {
                     FBUtilities.waitOnFutures(flushes);
                 }
-                catch (Throwable e)
+                catch (Throwable t)
                 {
+                    JVMStabilityInspector.inspectThrowable(t);
                     // don't let this stop us from shutting down the commitlog and other
thread pools
-                    logger.warn("Caught exception while waiting for memtable flushes during
shutdown hook", e);
+                    logger.warn("Caught exception while waiting for memtable flushes during
shutdown hook", t);
                 }
 
                 CommitLog.instance.shutdownBlocking();
@@ -658,6 +661,15 @@ public class StorageService extends NotificationBroadcasterSupport implements
IE
         }
     }
 
+    /**
+     * In the event of forceful termination we need to remove the shutdown hook to prevent
hanging (OOM for instance)
+     */
+    public void removeShutdownHook()
+    {
+        if (drainOnShutdown != null)
+            Runtime.getRuntime().removeShutdownHook(drainOnShutdown);
+    }
+
     private boolean shouldBootstrap()
     {
         return DatabaseDescriptor.isAutoBootstrap() && !SystemKeyspace.bootstrapComplete()
&& !DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/ConnectionHandler.java b/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
index a20eb34..6092046 100644
--- a/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
+++ b/src/java/org/apache/cassandra/streaming/ConnectionHandler.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
 import org.apache.cassandra.streaming.messages.StreamInitMessage;
 import org.apache.cassandra.streaming.messages.StreamMessage;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * ConnectionHandler manages incoming/outgoing message exchange for the {@link StreamSession}.
@@ -256,9 +257,10 @@ public class ConnectionHandler
                 // socket is closed
                 close();
             }
-            catch (Throwable e)
+            catch (Throwable t)
             {
-                session.onError(e);
+                JVMStabilityInspector.inspectThrowable(t);
+                session.onError(t);
             }
             finally
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
index 672cf2b..8569b88 100644
--- a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
@@ -27,6 +27,7 @@ import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
 import org.apache.cassandra.streaming.StreamReader;
 import org.apache.cassandra.streaming.StreamSession;
 import org.apache.cassandra.streaming.compress.CompressedStreamReader;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * IncomingFileMessage is used to receive the part(or whole) of a SSTable data file.
@@ -46,9 +47,10 @@ public class IncomingFileMessage extends StreamMessage
             {
                 return new IncomingFileMessage(reader.read(in), header);
             }
-            catch (Throwable e)
+            catch (Throwable t)
             {
-                session.doRetry(header, e);
+                JVMStabilityInspector.inspectThrowable(t);
+                session.doRetry(header, t);
                 return null;
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 3eab4e3..7faec72 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -60,6 +60,7 @@ import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.streaming.StreamManagerMBean;
 import org.apache.cassandra.streaming.management.StreamStateCompositeData;
 import org.apache.cassandra.utils.concurrent.SimpleCondition;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * JMX client operations for Cassandra.
@@ -264,9 +265,10 @@ public class NodeProbe implements AutoCloseable
                 ssProxy.removeNotificationListener(runner);
                 jmxc.removeConnectionNotificationListener(runner);
             }
-            catch (Throwable e) 
+            catch (Throwable t)
             {
-                out.println("Exception occurred during clean-up. " + e);
+                JVMStabilityInspector.inspectThrowable(t);
+                out.println("Exception occurred during clean-up. " + t);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index 4c5e101..2b273c5 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -30,6 +30,7 @@ import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.metadata.MetadataType;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * Reset level to 0 on a given set of sstables
@@ -100,9 +101,10 @@ public class SSTableLevelResetter
                 out.println("Found no sstables, did you give the correct keyspace/columnfamily?");
             }
         }
-        catch (Throwable e)
+        catch (Throwable t)
         {
-            e.printStackTrace();
+            JVMStabilityInspector.inspectThrowable(t);
+            t.printStackTrace();
             System.exit(1);
         }
         System.exit(0);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/transport/Message.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/transport/Message.java b/src/java/org/apache/cassandra/transport/Message.java
index f4c6b31..9efc424 100644
--- a/src/java/org/apache/cassandra/transport/Message.java
+++ b/src/java/org/apache/cassandra/transport/Message.java
@@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.transport.messages.*;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 /**
  * A message from the CQL binary protocol.
@@ -440,10 +441,11 @@ public abstract class Message
                 response.attach(connection);
                 connection.applyStateTransition(request.type, response.type);
             }
-            catch (Throwable ex)
+            catch (Throwable t)
             {
+                JVMStabilityInspector.inspectThrowable(t);
                 UnexpectedChannelExceptionHandler handler = new UnexpectedChannelExceptionHandler(ctx.channel(),
true);
-                flush(new FlushItem(ctx, ErrorMessage.fromException(ex, handler).setStreamId(request.getStreamId()),
request.getSourceFrame()));
+                flush(new FlushItem(ctx, ErrorMessage.fromException(t, handler).setStreamId(request.getStreamId()),
request.getSourceFrame()));
                 return;
             }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/utils/CLibrary.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/CLibrary.java b/src/java/org/apache/cassandra/utils/CLibrary.java
index 1d3c014..1927578 100644
--- a/src/java/org/apache/cassandra/utils/CLibrary.java
+++ b/src/java/org/apache/cassandra/utils/CLibrary.java
@@ -312,6 +312,7 @@ public final class CLibrary
         }
         catch (Throwable t)
         {
+            JVMStabilityInspector.inspectThrowable(t);
             // ignore
             return -1;
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/utils/FastByteOperations.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/FastByteOperations.java b/src/java/org/apache/cassandra/utils/FastByteOperations.java
index 1b58b66..6e25492 100644
--- a/src/java/org/apache/cassandra/utils/FastByteOperations.java
+++ b/src/java/org/apache/cassandra/utils/FastByteOperations.java
@@ -117,7 +117,9 @@ public class FastByteOperations
                 return comparer;
             }
             catch (Throwable t)
-            { // ensure we really catch *everything*
+            {
+                JVMStabilityInspector.inspectThrowable(t);
+                // ensure we really catch *everything*
                 return new PureJavaOperations();
             }
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
new file mode 100644
index 0000000..9fdc5ea
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.cassandra.service.StorageService;
+
+public class JVMStabilityInspector
+{
+    private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
+    /**
+     * Certain Throwables and Exceptions represent "Stop" conditions for the server.
+     * @param t
+     *      The Throwable to check for server-stop conditions
+     */
+    public static void inspectThrowable(Throwable t)
+    {
+        boolean isUnstable = false;
+        if (t instanceof OutOfMemoryError)
+            isUnstable = true;
+        if (isUnstable)
+        {
+            t.printStackTrace(System.err);
+            logger.error("JVM state determined to be unstable.  Exiting forcefully due to:",
t);
+            StorageService.instance.removeShutdownHook();
+            System.exit(100);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/14521347/src/java/org/apache/cassandra/utils/ResourceWatcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/ResourceWatcher.java b/src/java/org/apache/cassandra/utils/ResourceWatcher.java
index ac695a7..2dfab95 100644
--- a/src/java/org/apache/cassandra/utils/ResourceWatcher.java
+++ b/src/java/org/apache/cassandra/utils/ResourceWatcher.java
@@ -60,6 +60,7 @@ public class ResourceWatcher
             }
             catch (Throwable t)
             {
+                JVMStabilityInspector.inspectThrowable(t);
                 logger.error(String.format("Timed run of %s failed.", callback.getClass()),
t);
             }
         }


Mime
View raw message