Return-Path: X-Original-To: apmail-tomcat-dev-archive@www.apache.org Delivered-To: apmail-tomcat-dev-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 38E296AD2 for ; Tue, 7 Jun 2011 23:47:44 +0000 (UTC) Received: (qmail 7737 invoked by uid 500); 7 Jun 2011 23:47:43 -0000 Delivered-To: apmail-tomcat-dev-archive@tomcat.apache.org Received: (qmail 7673 invoked by uid 500); 7 Jun 2011 23:47:43 -0000 Mailing-List: contact dev-help@tomcat.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: "Tomcat Developers List" Delivered-To: mailing list dev@tomcat.apache.org Received: (qmail 7664 invoked by uid 99); 7 Jun 2011 23:47:43 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Jun 2011 23:47:43 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Jun 2011 23:47:38 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id DD7A5238890D; Tue, 7 Jun 2011 23:47:16 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1133200 - /tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java Date: Tue, 07 Jun 2011 23:47:16 -0000 To: dev@tomcat.apache.org From: markt@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110607234716.DD7A5238890D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: markt Date: Tue Jun 7 23:47:16 2011 New Revision: 1133200 URL: http://svn.apache.org/viewvc?rev=1133200&view=rev Log: Enable for async requests. Looking for stuck threads - no special async support required here. When the asycn timeout is infinite requests may get stuck but that'll need a different detection mechanism. Modified: tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java Modified: tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java?rev=1133200&r1=1133199&r2=1133200&view=diff ============================================================================== --- tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java (original) +++ tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java Tue Jun 7 23:47:16 2011 @@ -1,302 +1,311 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.catalina.valves; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.servlet.ServletException; - -import org.apache.catalina.LifecycleException; -import org.apache.catalina.connector.Request; -import org.apache.catalina.connector.Response; -import org.apache.juli.logging.Log; -import org.apache.juli.logging.LogFactory; -import org.apache.tomcat.util.res.StringManager; - -/** - * This valve allows to detect requests that take a long time to process, which might - * indicate that the thread that is processing it is stuck. - * Based on code proposed by TomLu in Bugzilla entry #50306 - * - * @author slaurent - * - */ -public class StuckThreadDetectionValve extends ValveBase { - - /** - * The descriptive information related to this implementation. - */ - private static final String info = - "org.apache.catalina.valves.StuckThreadDetectionValve/1.0"; - /** - * Logger - */ - private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class); - - /** - * The string manager for this package. - */ - private static final StringManager sm = - StringManager.getManager(Constants.Package); - - /** - * Keeps count of the number of stuck threads detected - */ - private final AtomicInteger stuckCount = new AtomicInteger(0); - - /** - * In seconds. Default 600 (10 minutes). - */ - private int threshold = 600; - - /** - * The only references we keep to actual running Thread objects are in - * this Map (which is automatically cleaned in invoke()s finally clause). - * That way, Threads can be GC'ed, eventhough the Valve still thinks they - * are stuck (caused by a long monitor interval) - */ - private ConcurrentHashMap activeThreads = - new ConcurrentHashMap(); - /** - * - */ - private Queue completedStuckThreadsQueue = - new ConcurrentLinkedQueue(); - - /** - * Specify the threshold (in seconds) used when checking for stuck threads. - * If <=0, the detection is disabled. The default is 600 seconds. - * - * @param threshold - * The new threshold in seconds - */ - public void setThreshold(int threshold) { - this.threshold = threshold; - } - - /** - * @see #setThreshold(int) - * @return The current threshold in seconds - */ - public int getThreshold() { - return threshold; - } - - @Override - protected void initInternal() throws LifecycleException { - super.initInternal(); - - if (log.isDebugEnabled()) { - log.debug("Monitoring stuck threads with threshold = " - + threshold - + " sec"); - } - } - - /** - * Return descriptive information about this Valve implementation. - */ - @Override - public String getInfo() { - return info; - } - - private void notifyStuckThreadDetected(MonitoredThread monitoredThread, - long activeTime, int numStuckThreads) { - if (log.isWarnEnabled()) { - String msg = sm.getString( - "stuckThreadDetectionValve.notifyStuckThreadDetected", - monitoredThread.getThread().getName(), activeTime, - monitoredThread.getStartTime(), numStuckThreads, - monitoredThread.getRequestUri(), threshold); - // msg += "\n" + getStackTraceAsString(trace); - Throwable th = new Throwable(); - th.setStackTrace(monitoredThread.getThread().getStackTrace()); - log.warn(msg, th); - } - } - - private void notifyStuckThreadCompleted(String threadName, - long activeTime, int numStuckThreads) { - if (log.isWarnEnabled()) { - String msg = sm.getString( - "stuckThreadDetectionValve.notifyStuckThreadCompleted", - threadName, activeTime, numStuckThreads); - // Since the "stuck thread notification" is warn, this should also - // be warn - log.warn(msg); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void invoke(Request request, Response response) - throws IOException, ServletException { - - if (threshold <= 0) { - // short-circuit if not monitoring stuck threads - getNext().invoke(request, response); - return; - } - - // Save the thread/runnable - // Keeping a reference to the thread object here does not prevent - // GC'ing, as the reference is removed from the Map in the finally clause - - Long key = new Long(Thread.currentThread().getId()); - StringBuffer requestUrl = request.getRequestURL(); - if(request.getQueryString()!=null) { - requestUrl.append("?"); - requestUrl.append(request.getQueryString()); - } - MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(), - requestUrl.toString()); - activeThreads.put(key, monitoredThread); - - try { - getNext().invoke(request, response); - } finally { - activeThreads.remove(key); - if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) { - completedStuckThreadsQueue.add( - new CompletedStuckThread(monitoredThread.getThread().getName(), - monitoredThread.getActiveTimeInMillis())); - } - } - } - - @Override - public void backgroundProcess() { - super.backgroundProcess(); - - long thresholdInMillis = threshold * 1000; - - // Check monitored threads, being careful that the request might have - // completed by the time we examine it - for (MonitoredThread monitoredThread : activeThreads.values()) { - long activeTime = monitoredThread.getActiveTimeInMillis(); - - if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) { - int numStuckThreads = stuckCount.incrementAndGet(); - notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads); - } - } - // Check if any threads previously reported as stuck, have finished. - for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll(); - completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) { - - int numStuckThreads = stuckCount.decrementAndGet(); - notifyStuckThreadCompleted(completedStuckThread.getName(), - completedStuckThread.getTotalActiveTime(), numStuckThreads); - } - } - - public long[] getStuckThreadIds() { - List idList = new ArrayList(); - for (MonitoredThread monitoredThread : activeThreads.values()) { - if (monitoredThread.isMarkedAsStuck()) { - idList.add(monitoredThread.getThread().getId()); - } - } - - long[] result = new long[idList.size()]; - for (int i = 0; i < result.length; i++) { - result[i] = idList.get(i); - } - return result; - } - - private class MonitoredThread { - - /** - * Reference to the thread to get a stack trace from background task - */ - private final Thread thread; - private final String requestUri; - private final long start; - private final AtomicInteger state = new AtomicInteger( - MonitoredThreadState.RUNNING.ordinal()); - - public MonitoredThread(Thread thread, String requestUri) { - this.thread = thread; - this.requestUri = requestUri; - this.start = System.currentTimeMillis(); - } - - public Thread getThread() { - return this.thread; - } - - public String getRequestUri() { - return requestUri; - } - - public long getActiveTimeInMillis() { - return System.currentTimeMillis() - start; - } - - public Date getStartTime() { - return new Date(start); - } - - public boolean markAsStuckIfStillRunning() { - return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(), - MonitoredThreadState.STUCK.ordinal()); - } - - public MonitoredThreadState markAsDone() { - int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal()); - return MonitoredThreadState.values()[val]; - } - - boolean isMarkedAsStuck() { - return this.state.get() == MonitoredThreadState.STUCK.ordinal(); - } - } - - private class CompletedStuckThread { - - private String threadName; - private long totalActiveTime; - - public CompletedStuckThread(String threadName, long totalActiveTime) { - this.threadName = threadName; - this.totalActiveTime = totalActiveTime; - } - - public String getName() { - return this.threadName; - } - - public long getTotalActiveTime() { - return this.totalActiveTime; - } - } - - private enum MonitoredThreadState { - RUNNING, STUCK, DONE; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.catalina.valves; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.servlet.ServletException; + +import org.apache.catalina.LifecycleException; +import org.apache.catalina.connector.Request; +import org.apache.catalina.connector.Response; +import org.apache.juli.logging.Log; +import org.apache.juli.logging.LogFactory; +import org.apache.tomcat.util.res.StringManager; + +/** + * This valve allows to detect requests that take a long time to process, which might + * indicate that the thread that is processing it is stuck. + * Based on code proposed by TomLu in Bugzilla entry #50306 + * + * @author slaurent + * + */ +public class StuckThreadDetectionValve extends ValveBase { + + /** + * The descriptive information related to this implementation. + */ + private static final String info = + "org.apache.catalina.valves.StuckThreadDetectionValve/1.0"; + /** + * Logger + */ + private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class); + + /** + * The string manager for this package. + */ + private static final StringManager sm = + StringManager.getManager(Constants.Package); + + /** + * Keeps count of the number of stuck threads detected + */ + private final AtomicInteger stuckCount = new AtomicInteger(0); + + /** + * In seconds. Default 600 (10 minutes). + */ + private int threshold = 600; + + /** + * The only references we keep to actual running Thread objects are in + * this Map (which is automatically cleaned in invoke()s finally clause). + * That way, Threads can be GC'ed, eventhough the Valve still thinks they + * are stuck (caused by a long monitor interval) + */ + private ConcurrentHashMap activeThreads = + new ConcurrentHashMap(); + /** + * + */ + private Queue completedStuckThreadsQueue = + new ConcurrentLinkedQueue(); + + /** + * Specify the threshold (in seconds) used when checking for stuck threads. + * If <=0, the detection is disabled. The default is 600 seconds. + * + * @param threshold + * The new threshold in seconds + */ + public void setThreshold(int threshold) { + this.threshold = threshold; + } + + /** + * @see #setThreshold(int) + * @return The current threshold in seconds + */ + public int getThreshold() { + return threshold; + } + + + /** + * Required to enable async support. + */ + public StuckThreadDetectionValve() { + super(true); + } + + + @Override + protected void initInternal() throws LifecycleException { + super.initInternal(); + + if (log.isDebugEnabled()) { + log.debug("Monitoring stuck threads with threshold = " + + threshold + + " sec"); + } + } + + /** + * Return descriptive information about this Valve implementation. + */ + @Override + public String getInfo() { + return info; + } + + private void notifyStuckThreadDetected(MonitoredThread monitoredThread, + long activeTime, int numStuckThreads) { + if (log.isWarnEnabled()) { + String msg = sm.getString( + "stuckThreadDetectionValve.notifyStuckThreadDetected", + monitoredThread.getThread().getName(), activeTime, + monitoredThread.getStartTime(), numStuckThreads, + monitoredThread.getRequestUri(), threshold); + // msg += "\n" + getStackTraceAsString(trace); + Throwable th = new Throwable(); + th.setStackTrace(monitoredThread.getThread().getStackTrace()); + log.warn(msg, th); + } + } + + private void notifyStuckThreadCompleted(String threadName, + long activeTime, int numStuckThreads) { + if (log.isWarnEnabled()) { + String msg = sm.getString( + "stuckThreadDetectionValve.notifyStuckThreadCompleted", + threadName, activeTime, numStuckThreads); + // Since the "stuck thread notification" is warn, this should also + // be warn + log.warn(msg); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void invoke(Request request, Response response) + throws IOException, ServletException { + + if (threshold <= 0) { + // short-circuit if not monitoring stuck threads + getNext().invoke(request, response); + return; + } + + // Save the thread/runnable + // Keeping a reference to the thread object here does not prevent + // GC'ing, as the reference is removed from the Map in the finally clause + + Long key = new Long(Thread.currentThread().getId()); + StringBuffer requestUrl = request.getRequestURL(); + if(request.getQueryString()!=null) { + requestUrl.append("?"); + requestUrl.append(request.getQueryString()); + } + MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(), + requestUrl.toString()); + activeThreads.put(key, monitoredThread); + + try { + getNext().invoke(request, response); + } finally { + activeThreads.remove(key); + if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) { + completedStuckThreadsQueue.add( + new CompletedStuckThread(monitoredThread.getThread().getName(), + monitoredThread.getActiveTimeInMillis())); + } + } + } + + @Override + public void backgroundProcess() { + super.backgroundProcess(); + + long thresholdInMillis = threshold * 1000; + + // Check monitored threads, being careful that the request might have + // completed by the time we examine it + for (MonitoredThread monitoredThread : activeThreads.values()) { + long activeTime = monitoredThread.getActiveTimeInMillis(); + + if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) { + int numStuckThreads = stuckCount.incrementAndGet(); + notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads); + } + } + // Check if any threads previously reported as stuck, have finished. + for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll(); + completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) { + + int numStuckThreads = stuckCount.decrementAndGet(); + notifyStuckThreadCompleted(completedStuckThread.getName(), + completedStuckThread.getTotalActiveTime(), numStuckThreads); + } + } + + public long[] getStuckThreadIds() { + List idList = new ArrayList(); + for (MonitoredThread monitoredThread : activeThreads.values()) { + if (monitoredThread.isMarkedAsStuck()) { + idList.add(monitoredThread.getThread().getId()); + } + } + + long[] result = new long[idList.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = idList.get(i); + } + return result; + } + + private class MonitoredThread { + + /** + * Reference to the thread to get a stack trace from background task + */ + private final Thread thread; + private final String requestUri; + private final long start; + private final AtomicInteger state = new AtomicInteger( + MonitoredThreadState.RUNNING.ordinal()); + + public MonitoredThread(Thread thread, String requestUri) { + this.thread = thread; + this.requestUri = requestUri; + this.start = System.currentTimeMillis(); + } + + public Thread getThread() { + return this.thread; + } + + public String getRequestUri() { + return requestUri; + } + + public long getActiveTimeInMillis() { + return System.currentTimeMillis() - start; + } + + public Date getStartTime() { + return new Date(start); + } + + public boolean markAsStuckIfStillRunning() { + return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(), + MonitoredThreadState.STUCK.ordinal()); + } + + public MonitoredThreadState markAsDone() { + int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal()); + return MonitoredThreadState.values()[val]; + } + + boolean isMarkedAsStuck() { + return this.state.get() == MonitoredThreadState.STUCK.ordinal(); + } + } + + private class CompletedStuckThread { + + private String threadName; + private long totalActiveTime; + + public CompletedStuckThread(String threadName, long totalActiveTime) { + this.threadName = threadName; + this.totalActiveTime = totalActiveTime; + } + + public String getName() { + return this.threadName; + } + + public long getTotalActiveTime() { + return this.totalActiveTime; + } + } + + private enum MonitoredThreadState { + RUNNING, STUCK, DONE; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org For additional commands, e-mail: dev-help@tomcat.apache.org