jackrabbit-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rabbeet <wiel...@trivantisdev.com>
Subject Deadlock JR 1.6.4, 2.2, 2.2.4 involving CachingHierarchyManager
Date Thu, 17 Mar 2011 02:50:22 GMT
  We have been hunting down a deadlock problem with our use of jackrabbit. In
our limited understanding it looks like the contention is between the
workspace SISM and the version manager SISM when caching is involved.
  
  We tracked the problem to an area of code that referenced this fix:
  https://issues.apache.org/jira/browse/JCR-2171
  
  We found that jira item a fitting description of the deadlock but in our
case downgrading the SISM write lock is not enough to clear the contention
because Thread 6 below is trying to aquire a write lock, while Thread 20
still holds the downgraded write lock.
  
  ** Note: the thread stacks contain (v2.2.4) DefaultISMLocking classes with
instrumentation so line numbers are off, also the thread representations are
really two stacks placed together one when acquiring and one when waiting.

**** Thread 20
  
  ----------------------------------  when waiting on VERSION MANAGER SISM
  java.lang.Object.wait(Native Method)
  java.lang.Object.wait(Object.java:485)
 
org.apache.jackrabbit.core.state.DefaultISMLocking.acquireReadLock(DefaultISMLocking.java:178)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.acquireReadLock(SharedItemStateManager.java:1834)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.hasItemState(SharedItemStateManager.java:294)
 
org.apache.jackrabbit.core.version.VersionItemStateProvider.hasItemState(VersionItemStateProvider.java:140)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.hasItemState(SharedItemStateManager.java:314)
 
org.apache.jackrabbit.core.CachingHierarchyManager.stateDiscarded(CachingHierarchyManager.java:362)
 
org.apache.jackrabbit.core.state.StateChangeDispatcher.notifyStateDiscarded(StateChangeDispatcher.java:133)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.stateDiscarded(SharedItemStateManager.java:436)
 
org.apache.jackrabbit.core.state.ItemState.notifyStateDiscarded(ItemState.java:222)
  org.apache.jackrabbit.core.state.ItemState.discard(ItemState.java:360)
  org.apache.jackrabbit.core.state.ChangeLog.persisted(ChangeLog.java:305)
 
org.apache.jackrabbit.core.state.SharedItemStateManager$Update.end(SharedItemStateManager.java:777)
 
org.apache.jackrabbit.core.state.XAItemStateManager.commit(XAItemStateManager.java:181)
 
org.apache.jackrabbit.core.TransactionContext.commit(TransactionContext.java:209)
  org.apache.jackrabbit.core.XASessionImpl.commit(XASessionImpl.java:327)
 
com.co.prod.transaction.TransactionMgr.commitTransaction(TransactionMgr.java:145)
 
com.co.prod.transaction.TransactionMgr.commitTransaction(TransactionMgr.java:115)
 
com.co.prod.WebtoraWebContext.commitTransaction(WebtoraWebContext.java:305)
  com.co.prod.swr.SWRServlet.rexec(SWRServlet.java:590)
  com.co.prod.swr.SWRServlet.executeRequest(SWRServlet.java:441)
  com.co.prod.swr.SWRServlet.service(SWRServlet.java:236)
  javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
  com.co.util.BrowserCacheFilter.doFilter(BrowserCacheFilter.java:137)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
com.co.prod.WebtoraWebContextFilter.doFilter(WebtoraWebContextFilter.java:33)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
 
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
 
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
 
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
 
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
 
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:286)
  org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190)
  org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
  org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:767)
 
org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:697)
 
org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:889)
  
  -----------------------------------  when aquired WORKSPACE SISM LOCK
  java.lang.Thread.getStackTrace(Thread.java:1436)
 
org.apache.jackrabbit.core.state.DefaultISMLocking.acquireWriteLock(DefaultISMLocking.java:308)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.acquireWriteLock(SharedItemStateManager.java:1848)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.access$200(SharedItemStateManager.java:113)
 
org.apache.jackrabbit.core.state.SharedItemStateManager$Update.begin(SharedItemStateManager.java:563)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.beginUpdate(SharedItemStateManager.java:1457)
 
org.apache.jackrabbit.core.state.XAItemStateManager.prepare(XAItemStateManager.java:163)
 
org.apache.jackrabbit.core.TransactionContext.prepare(TransactionContext.java:157)
  org.apache.jackrabbit.core.XASessionImpl.prepare(XASessionImpl.java:312)
 
com.co.prod.transaction.TransactionMgr.commitTransaction(TransactionMgr.java:115)
 
com.co.prod.WebtoraWebContext.commitTransaction(WebtoraWebContext.java:305)
  com.co.prod.swr.SWRServlet.rexec(SWRServlet.java:590)
  com.co.prod.swr.SWRServlet.executeRequest(SWRServlet.java:441)
  com.co.prod.swr.SWRServlet.service(SWRServlet.java:236)
  javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
  com.co.util.BrowserCacheFilter.doFilter(BrowserCacheFilter.java:137)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
com.co.prod.WebtoraWebContextFilter.doFilter(WebtoraWebContextFilter.java:33)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
 
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
 
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
 
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
 
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
 
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:286)
  org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190)
  org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
  org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:767)
 
org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:697)
 
org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:889)
  
  
  **** Thread 6
  
  -----------------------------------  when waiting on  WORKSPACE SISM LOCK
  java.lang.Object.wait(Native Method)
  java.lang.Object.wait(Object.java:485)
 
org.apache.jackrabbit.core.state.DefaultISMLocking.acquireWriteLock(DefaultISMLocking.java:278)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.acquireWriteLock(SharedItemStateManager.java:1848)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.access$200(SharedItemStateManager.java:113)
 
org.apache.jackrabbit.core.state.SharedItemStateManager$Update.begin(SharedItemStateManager.java:563)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.beginUpdate(SharedItemStateManager.java:1457)
 
org.apache.jackrabbit.core.state.XAItemStateManager.prepare(XAItemStateManager.java:163)
 
org.apache.jackrabbit.core.TransactionContext.prepare(TransactionContext.java:157)
  org.apache.jackrabbit.core.XASessionImpl.prepare(XASessionImpl.java:312)
 
com.co.prod.transaction.TransactionMgr.commitTransaction(TransactionMgr.java:115)
 
com.co.prod.WebtoraWebContext.commitTransaction(WebtoraWebContext.java:305)
  com.co.prod.swr.SWRServlet.rexec(SWRServlet.java:590)
  com.co.prod.swr.SWRServlet.executeRequest(SWRServlet.java:441)
  com.co.prod.swr.SWRServlet.service(SWRServlet.java:236)
  javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
  com.co.util.BrowserCacheFilter.doFilter(BrowserCacheFilter.java:137)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
com.co.prod.WebtoraWebContextFilter.doFilter(WebtoraWebContextFilter.java:33)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
 
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
 
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
 
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
 
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
 
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:286)
  org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190)
  org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
  org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:767)
 
org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:697)
 
org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:889)
 
org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:690)
  java.lang.Thread.run(Thread.java:619)
  
  
 ----------------------------------  when aquired VERSION MANAGER SISM
  java.lang.Thread.getStackTrace(Thread.java:1436)
 
org.apache.jackrabbit.core.state.DefaultISMLocking.acquireWriteLock(DefaultISMLocking.java:308)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.acquireWriteLock(SharedItemStateManager.java:1848)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.access$200(SharedItemStateManager.java:113)
 
org.apache.jackrabbit.core.state.SharedItemStateManager$Update.begin(SharedItemStateManager.java:563)
 
org.apache.jackrabbit.core.state.SharedItemStateManager.beginUpdate(SharedItemStateManager.java:1457)
 
org.apache.jackrabbit.core.state.XAItemStateManager.prepare(XAItemStateManager.java:163)
 
org.apache.jackrabbit.core.version.InternalXAVersionManager.prepare(InternalXAVersionManager.java:582)
 
org.apache.jackrabbit.core.TransactionContext.prepare(TransactionContext.java:157)
  org.apache.jackrabbit.core.XASessionImpl.prepare(XASessionImpl.java:312)
 
com.co.prod.transaction.TransactionMgr.commitTransaction(TransactionMgr.java:115)
 
com.co.prod.WebtoraWebContext.commitTransaction(WebtoraWebContext.java:305)
  com.co.prod.swr.SWRServlet.rexec(SWRServlet.java:590)
  com.co.prod.swr.SWRServlet.executeRequest(SWRServlet.java:441)
  com.co.prod.swr.SWRServlet.service(SWRServlet.java:236)
  javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
  com.co.util.BrowserCacheFilter.doFilter(BrowserCacheFilter.java:137)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
com.co.prod.WebtoraWebContextFilter.doFilter(WebtoraWebContextFilter.java:33)
 
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
 
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
 
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
 
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
 
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
 
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
 
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
 
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:286)
  org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:190)
  org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
  org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:767)
 
org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:697)
 
org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:889)
 
org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:690)
  java.lang.Thread.run(Thread.java:619)
  
  
Our knowledge of the internals of JR are limited and we are hoping for some
help in finding a way to avoid this deadlock situation.  We have to restart
our web application two to three times a week so we are testing a patch to
get us by.
  
The patch is to modify CachingHierarchyManager so it does not query the item
state and cause the deadlock:
  
    public void stateDiscarded(ItemState discarded) {
        if (discarded.isTransient() && !discarded.hasOverlayedState()
                && discarded.getStatus() == ItemState.STATUS_NEW) {
            // a new node has been discarded -> remove from cache
            evictAll(discarded.getId(), true);
        ////////} else if (provider.hasItemState(discarded.getId())) {
		} else {
            evictAll(discarded.getId(), false);
        ////////} else {
        ////////   evictAll(discarded.getId(), true);
        }
    }
	

Thanks in advance for any help.
	
	


--
View this message in context: http://jackrabbit.510166.n4.nabble.com/Deadlock-JR-1-6-4-2-2-2-2-4-involving-CachingHierarchyManager-tp3383628p3383628.html
Sent from the Jackrabbit - Dev mailing list archive at Nabble.com.

Mime
View raw message