#927 [HHQ-1651] Deadlock in EmailFilter

[Hyperic Team]

http://jira.hyperic.com/browse/HHQ-1651
In an environment with many alerts configured to 'Filter alerts for related resources' it's possible for the EmailFilter to deadlock.

Many threads are stuck at:

    Thread[JMS SessionPool Worker-22302,5,ASF Session Pool Threads]


     org.hyperic.hq.bizapp.server.action.email.EmailFilter.scheduleJob(EmailFilter.java:349)


     org.hyperic.hq.bizapp.server.action.email.EmailFilter.sendAlert(EmailFilter.java:205)


     org.hyperic.hq.bizapp.server.action.email.EmailAction.execute(EmailAction.java:172)

Which is peculiar since I'd expect these threads to be stuck on line 347, the beginning of the synchronized block.

The one thread inside the synchronized block is doing a schedule within Quartz:

    Thread[JMS SessionPool Worker-22283,5,ASF Session Pool Threads]


     java.net.SocketInputStream.socketRead0(Native Method)


     java.net.SocketInputStream.read(Unknown Source)


     org.postgresql.core.VisibleBufferedInputStream.readMore(VisibleBufferedInputStream.java:135)


     org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:104)


     org.postgresql.core.VisibleBufferedInputStream.read(VisibleBufferedInputStream.java:73)


     org.postgresql.core.PGStream.ReceiveChar(PGStream.java:255)


     org.postgresql.core.v2.QueryExecutorImpl.processResults(QueryExecutorImpl.java:408)


     org.postgresql.core.v2.QueryExecutorImpl.execute(QueryExecutorImpl.java:364)


     org.postgresql.core.v2.QueryExecutorImpl.execute(QueryExecutorImpl.java:258)


     org.postgresql.jdbc2.AbstractJdbc2Statement.execute(AbstractJdbc2Statement.java:452)


     org.postgresql.jdbc2.AbstractJdbc2Statement.executeWithFlags(AbstractJdbc2Statement.java:351)


     org.postgresql.jdbc2.AbstractJdbc2Statement.executeQuery(AbstractJdbc2Statement.java:255)


     org.jboss.resource.adapter.jdbc.CachedPreparedStatement.executeQuery(CachedPreparedStatement.java:76)


     org.jboss.resource.adapter.jdbc.WrappedPreparedStatement.executeQuery(WrappedPreparedStatement.java:211)


     org.quartz.impl.jdbcjobstore.StdRowLockSemaphore.obtainLock(StdRowLockSemaphore.java:137)


     org.quartz.impl.jdbcjobstore.JobStoreCMT.storeJobAndTrigger(JobStoreCMT.java:259)


     org.quartz.core.QuartzScheduler.scheduleJob(QuartzScheduler.java:613)


     org.quartz.impl.StdScheduler.scheduleJob(StdScheduler.java:238)


     org.hyperic.hq.scheduler.server.mbean.SchedulerService.scheduleJob(SchedulerService.java:199)


     sun.reflect.GeneratedMethodAccessor2839.invoke(Unknown Source)


     sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)


     java.lang.reflect.Method.invoke(Unknown Source)


     org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)


     org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)


     org.jboss.mx.server.Invocation.invoke(Invocation.java:72)


     org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)


     org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)


     org.jboss.mx.util.JMXInvocationHandler.invoke(JMXInvocationHandler.java:273)


     $Proxy370.scheduleJob(Unknown Source)


     org.hyperic.hq.scheduler.server.session.SchedulerEJBImpl.scheduleJob(SchedulerEJBImpl.java:309)


     sun.reflect.GeneratedMethodAccessor2838.invoke(Unknown Source)


     sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)


     java.lang.reflect.Method.invoke(Unknown Source)


     org.jboss.invocation.Invocation.performCall(Invocation.java:345)


     org.jboss.ejb.StatelessSessionContainer$ContainerInterceptor.invoke(StatelessSessionContainer.java:214)


     org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:149)


     org.jboss.ejb.plugins.StatelessSessionInstanceInterceptor.invoke(StatelessSessionInstanceInterceptor.java:154)


     org.jboss.webservice.server.ServiceEndpointInterceptor.invoke(ServiceEndpointInterceptor.java:54)


     org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:48)


     org.hyperic.hq.application.HQApp$Snatcher.invokeNextBoth(HQApp.java:408)


     org.hyperic.hq.application.HQApp$Snatcher.invokeNext(HQApp.java:460)


     org.hyperic.txsnatch.TxSnatch.invoke(TxSnatch.java:71)


     org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:106)


     org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:335)


     org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:166)


     org.hyperic.hq.application.HQApp$Snatcher.invokeNextBoth(HQApp.java:408)


     org.hyperic.hq.application.HQApp$Snatcher.invokeNext(HQApp.java:460)


     org.hyperic.txsnatch.TxSnatch.invoke(TxSnatch.java:71)


     org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:153)


     org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:192)


     org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:122)


     org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:624)


     org.jboss.ejb.Container.invoke(Container.java:873)


     org.jboss.ejb.plugins.local.BaseLocalProxyFactory.invoke(BaseLocalProxyFactory.java:415)


     org.jboss.ejb.plugins.local.StatelessSessionProxy.invoke(StatelessSessionProxy.java:88)


     $Proxy321.scheduleJob(Unknown Source)


     org.hyperic.hq.bizapp.server.action.email.EmailFilter.scheduleJob(EmailFilter.java:364)


     org.hyperic.hq.bizapp.server.action.email.EmailFilter.sendAlert(EmailFilter.java:205)


     org.hyperic.hq.bizapp.server.action.email.EmailAction.execute(EmailAction.java:172)