4 Replies Latest reply on Apr 4, 2008 9:27 AM by chipschoch

    Messaging gets hung up getting sockey

    chipschoch

      JBossAS 4.2.2.GA, JBM 1.4.0.SP1 with remoting 2.2.2.SP5

      We have several environments set up with 2 linux severs clustered and two windows servers that run a jms client program. In one environment we are consistently getting the system to hang when running a load test. Below is a stack trace. Once it gets hung up wating for a thread nothing else process messages because of the locks being held. Any ideas about what this means?

      Thread: Thread-126 : priority:5, demon:true, threadId:325, threadState:TIMED_WAITING, lockName:java.util.HashSet@42c94f
      
       java.lang.Object.wait(Native Method)
       org.jboss.remoting.transport.bisocket.BisocketClientInvoker.createSocket(BisocketClientInvoker.java:473)
       org.jboss.remoting.transport.socket.MicroSocketClientInvoker.getConnection(MicroSocketClientInvoker.java:801)
       org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:551)
       org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:418)
       org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:122)
       org.jboss.remoting.Client.invoke(Client.java:1634)
       org.jboss.remoting.Client.invoke(Client.java:548)
       org.jboss.remoting.Client.invokeOneway(Client.java:598)
       org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallback(ServerInvokerCallbackHandler.java:826)
       org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallbackOneway(ServerInvokerCallbackHandler.java:697)
       org.jboss.jms.server.endpoint.ServerSessionEndpoint.performDelivery(ServerSessionEndpoint.java:1490)
       org.jboss.jms.server.endpoint.ServerSessionEndpoint.handleDelivery(ServerSessionEndpoint.java:1375)
       org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:307)
       org.jboss.messaging.core.impl.RoundRobinDistributor.handle(RoundRobinDistributor.java:119)
       org.jboss.messaging.core.impl.MessagingQueue$DistributorWrapper.handle(MessagingQueue.java:582)
       org.jboss.messaging.core.impl.ClusterRoundRobinDistributor.handle(ClusterRoundRobinDistributor.java:79)
       org.jboss.messaging.core.impl.ChannelSupport.deliverInternal(ChannelSupport.java:476)
       org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:505)
       org.jboss.messaging.core.impl.ChannelSupport.handleInternal(ChannelSupport.java:628)
       org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:144)
       org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2195)
       org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489)
       org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:720)
       org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:401)
       org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87)
       org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java)
       org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157)
       sun.reflect.GeneratedMethodAccessor929.invoke(Unknown Source)
       sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
       java.lang.reflect.Method.invoke(Method.java:585)
       org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121)
       org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java)
       org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java)
       org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:90)
       org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143)
       org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809)
       org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815)
       org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63)
       org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826)
       org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758)
       org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101)
       org.jboss.remoting.Client.invoke(Client.java:1634)
       org.jboss.remoting.Client.invoke(Client.java:548)
       org.jboss.remoting.Client.invokeOneway(Client.java:598)
       org.jboss.remoting.Client.invokeOneway(Client.java:786)
       org.jboss.remoting.Client.invokeOneway(Client.java:776)
       org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:178)
       org.jboss.jms.client.delegate.DelegateSupport.doInvokeOneway(DelegateSupport.java:163)
       org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$send$aop(ClientSessionDelegate.java:478)
       org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java)
       org.jboss.jms.client.container.SessionAspect.handleSend(SessionAspect.java:632)
       org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect28.invoke(SessionAspect28.java)
       org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java)
       org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
       org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java)
       org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java)
       org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java)
       org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:266)
       org.jboss.aop.advice.org.jboss.jms.client.container.ProducerAspect39.invoke(ProducerAspect39.java)
       org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
       org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
       org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java)
       org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164)
       org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:207)
       org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145)
       com.eLynx.Messaging.MessageSender.sendTextMessage(MessageSender.java:410)
       com.eLynx.Messaging.MessageSender.sendXmlRequest(MessageSender.java:521)
       com.eLynx.Messaging.XmlMessageSender.request(XmlMessageSender.java:166)
       com.eLynx.Imaging.USignContainerImager.doImaging(USignContainerImager.java:194)
       com.eLynx.PackageProcessor.USignPackageProcessor.imagePackage(USignPackageProcessor.java:558)
       com.eLynx.PackageProcessor.USignPackageProcessor.executeState(USignPackageProcessor.java:440)
       com.eLynx.PackageProcessor.USignPackageProcessor.resume(USignPackageProcessor.java:907)
       com.eLynx.PackageProcessor.USignPackageProcessor.startPackageProcessing(USignPackageProcessor.java:1081)
       com.eLynx.Service.BpmExecutorMessageHandler.beginProcess(BpmExecutorMessageHandler.java:83)
       com.eLynx.Service.BpmExecutorMessageHandler.processMessage(BpmExecutorMessageHandler.java:230)
       com.eLynx.Messaging.MessageReceiverHandler.onMessage(MessageReceiverHandler.java:147)
       org.jboss.jms.client.container.ClientConsumer.callOnMessage(ClientConsumer.java:157)
       org.jboss.jms.client.container.ClientConsumer$ListenerRunner.run(ClientConsumer.java:965)
       EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:89)
       java.lang.Thread.run(Thread.java:595)


        • 1. Re: Messaging gets hung up getting sockey
          timfox

          Stack trace please?

          • 2. Re: Messaging gets hung up getting sockey
            chipschoch

            Isn't what I posted a stack trace? We have been looking into this and it seems that there is a firewall that is between the client and the server that drops connections that are inactive for a certain period of time.

            I am not sure what other stack trace you are asking for.

            • 3. Re: Messaging gets hung up getting sockey
              timfox

               

              "chip_schoch" wrote:
              Isn't what I posted a stack trace? We have been looking into this and it seems that there is a firewall that is between the client and the server that drops connections that are inactive for a certain period of time.

              I am not sure what other stack trace you are asking for.


              I'm looking for a thread dump of all stacks. What you provided was just one thread. You can get this by doing kill -3 <process id> on Linux or CTRL-BREAK in a console on windows.

              • 4. Re: Messaging gets hung up getting sockey
                chipschoch

                Well, our network folks changed a setting in a firewall between the servers and we have not seen it hang again. However, one of my colleagues saved a thread dump from the jmx-console, so here it is. Please don't ask me why there is a firewall between the servers in the first place, because that is a question I cannot answer. No one seems to be able to provide an answer.


                Total Threads: 238
                Total Thread Groups: 9
                Timestamp: 20080327091715
                
                Thread Group: system : max priority:10, demon:false
                
                 Thread: Reference Handler : priority:10, demon:true, threadId:2, threadState:WAITING, lockName:java.lang.ref.Reference$Lock@edb017
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
                
                 Thread: Finalizer : priority:8, demon:true, threadId:3, threadState:WAITING, lockName:java.lang.ref.ReferenceQueue$Lock@1f53392
                
                 java.lang.Object.wait(Native Method)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
                 java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
                
                 Thread: Signal Dispatcher : priority:9, demon:true, threadId:6, threadState:RUNNABLE, lockName:null
                 Thread: RMI TCP Accept-1098 : priority:5, demon:true, threadId:19, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 sun.rmi.transport.tcp.TCPTransport.run(TCPTransport.java:340)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI Reaper : priority:5, demon:false, threadId:21, threadState:WAITING, lockName:java.lang.ref.ReferenceQueue$Lock@8e0150
                
                 java.lang.Object.wait(Native Method)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
                 sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:336)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: GC Daemon : priority:2, demon:true, threadId:22, threadState:TIMED_WAITING, lockName:sun.misc.GC$LatencyLock@f83be2
                
                 java.lang.Object.wait(Native Method)
                 sun.misc.GC$Daemon.run(GC.java:100)
                
                 Thread: RMI TCP Accept-4444 : priority:5, demon:true, threadId:31, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 sun.rmi.transport.tcp.TCPTransport.run(TCPTransport.java:340)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI RenewClean-[app1.exp.elynx.net:1101] : priority:5, demon:true, threadId:67, threadState:TIMED_WAITING, lockName:java.lang.ref.ReferenceQueue$Lock@1da5b9e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
                 sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:501)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI TCP Accept-1101 : priority:5, demon:true, threadId:72, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 sun.rmi.transport.tcp.TCPTransport.run(TCPTransport.java:340)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI LeaseChecker : priority:5, demon:true, threadId:75, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 sun.rmi.transport.DGCImpl$LeaseChecker.run(DGCImpl.java:310)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI TCP Accept-4447 : priority:5, demon:true, threadId:78, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 sun.rmi.transport.tcp.TCPTransport.run(TCPTransport.java:340)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RMI RenewClean-[app3.exp.elynx.net:1101] : priority:5, demon:true, threadId:215, threadState:TIMED_WAITING, lockName:java.lang.ref.ReferenceQueue$Lock@a38d70
                
                 java.lang.Object.wait(Native Method)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
                 sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:501)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Java2D Disposer : priority:10, demon:true, threadId:302, threadState:WAITING, lockName:java.lang.ref.ReferenceQueue$Lock@19ac6da
                
                 java.lang.Object.wait(Native Method)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
                 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
                 sun.java2d.Disposer.run(Disposer.java:125)
                 java.lang.Thread.run(Thread.java:595)
                
                
                 Thread Group: main : max priority:10, demon:false
                
                 Thread: DestroyJavaVM : priority:5, demon:false, threadId:12, threadState:RUNNABLE, lockName:null
                
                 Thread Group: jboss : max priority:10, demon:false
                
                 Thread: Timer-0 : priority:5, demon:true, threadId:15, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@746338
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: ScannerThread : priority:5, demon:true, threadId:16, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.loop(AbstractDeploymentScanner.java:280)
                 org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.run(AbstractDeploymentScanner.java:225)
                
                 Thread: Timer-1 : priority:5, demon:true, threadId:20, threadState:WAITING, lockName:java.util.TaskQueue@167088e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 java.util.TimerThread.mainLoop(Timer.java:483)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: Listener:32787 : priority:5, demon:true, threadId:24, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 com.arjuna.ats.internal.arjuna.recovery.Listener.run(Listener.java:119)
                
                 Thread: Thread-4 : priority:5, demon:true, threadId:26, threadState:TIMED_WAITING, lockName:com.arjuna.ats.arjuna.coordinator.TransactionReaper@1a25a1d
                
                 java.lang.Object.wait(Native Method)
                 com.arjuna.ats.internal.arjuna.coordinator.ReaperThread.run(ReaperThread.java:98)
                
                 Thread: Thread-5 : priority:5, demon:true, threadId:27, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 com.arjuna.ats.internal.arjuna.recovery.ExpiredEntryMonitor.run(ExpiredEntryMonitor.java:237)
                
                 Thread: Listener:32790 : priority:5, demon:true, threadId:30, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 com.arjuna.ats.internal.arjuna.recovery.Listener.run(Listener.java:119)
                
                 Thread: PooledInvokerAcceptor#0-4445 : priority:5, demon:false, threadId:32, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jboss.invocation.pooled.server.PooledInvoker.run(PooledInvoker.java:262)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: AcceptorThread#0:4446 : priority:5, demon:false, threadId:33, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jboss.remoting.transport.socket.SocketServerInvoker.run(SocketServerInvoker.java:520)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ServerSocketRefresh : priority:5, demon:true, threadId:34, threadState:WAITING, lockName:java.lang.Object@1ca5f2d
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jboss.remoting.transport.socket.SocketServerInvoker$ServerSocketRefresh.run(SocketServerInvoker.java:704)
                
                 Thread: AcceptorThread#0:4457 : priority:5, demon:false, threadId:35, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jboss.remoting.transport.socket.SocketServerInvoker.run(SocketServerInvoker.java:520)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: secondaryServerSocketThread : priority:5, demon:true, threadId:36, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$SecondaryServerSocketThread.run(BisocketServerInvoker.java:889)
                
                 Thread: ServerSocketRefresh : priority:5, demon:true, threadId:37, threadState:WAITING, lockName:java.lang.Object@12e8967
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jboss.remoting.transport.socket.SocketServerInvoker$ServerSocketRefresh.run(SocketServerInvoker.java:704)
                
                 Thread: Timer-2 : priority:5, demon:true, threadId:38, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@3ed307
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: SnmpPortal--1 : priority:5, demon:false, threadId:49, threadState:WAITING, lockName:java.util.LinkedList@909bc6
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.opennms.protocols.snmp.SnmpPortal$Receiver.run(SnmpPortal.java:567)
                
                 Thread: Timer-3 : priority:5, demon:false, threadId:51, threadState:WAITING, lockName:java.util.TaskQueue@1e319cb
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 java.util.TimerThread.mainLoop(Timer.java:483)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: main-FastReceiver : priority:5, demon:false, threadId:50, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainDatagramSocketImpl.receive0(Native Method)
                 java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136)
                 java.net.DatagramSocket.receive(DatagramSocket.java:712)
                 org.opennms.protocols.snmp.SnmpPortal$1.run(SnmpPortal.java:467)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: SnmpPortal--1 : priority:5, demon:false, threadId:52, threadState:WAITING, lockName:java.util.LinkedList@b296c9
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.opennms.protocols.snmp.SnmpPortal$Receiver.run(SnmpPortal.java:567)
                
                 Thread: main-FastReceiver : priority:5, demon:false, threadId:53, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainDatagramSocketImpl.receive0(Native Method)
                 java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136)
                 java.net.DatagramSocket.receive(DatagramSocket.java:712)
                 org.opennms.protocols.snmp.SnmpPortal$1.run(SnmpPortal.java:467)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: SnmpTimer : priority:5, demon:true, threadId:54, threadState:TIMED_WAITING, lockName:java.util.TreeSet@c9939d
                
                 java.lang.Object.wait(Native Method)
                 org.opennms.protocols.snmp.SnmpTimer$TimerLogic.run(SnmpTimer.java:148)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: SnmpPortal--1 : priority:5, demon:false, threadId:55, threadState:WAITING, lockName:java.util.LinkedList@1fa437e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.opennms.protocols.snmp.SnmpPortal$Receiver.run(SnmpPortal.java:567)
                
                 Thread: main-FastReceiver : priority:5, demon:false, threadId:56, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainDatagramSocketImpl.receive0(Native Method)
                 java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136)
                 java.net.DatagramSocket.receive(DatagramSocket.java:712)
                 org.opennms.protocols.snmp.SnmpPortal$1.run(SnmpPortal.java:467)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Timer-4 : priority:5, demon:true, threadId:57, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@1651492
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: AsynchKeyChangeHandler Thread : priority:5, demon:false, threadId:70, threadState:WAITING, lockName:java.lang.Object@1298a44
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 org.jboss.ha.framework.server.AsynchEventHandler.run(AsynchEventHandler.java:102)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: AsynchViewChangeHandler Thread : priority:5, demon:false, threadId:71, threadState:WAITING, lockName:java.lang.Object@3494b4
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 org.jboss.ha.framework.server.AsynchEventHandler.run(AsynchEventHandler.java:102)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: PooledInvokerAcceptor#0-4448 : priority:5, demon:false, threadId:79, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jboss.invocation.pooled.server.PooledInvoker.run(PooledInvoker.java:262)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RequestController-1 : priority:10, demon:true, threadId:80, threadState:WAITING, lockName:java.lang.Object@ec7fd
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: ServerSocketListener : priority:5, demon:true, threadId:81, threadState:RUNNABLE, lockName:null
                
                 java.net.PlainSocketImpl.socketAccept(Native Method)
                 java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
                 java.net.ServerSocket.implAccept(ServerSocket.java:450)
                 java.net.ServerSocket.accept(ServerSocket.java:421)
                 org.jacorb.orb.iiop.IIOPListener$Acceptor.run(Unknown Source)
                
                 Thread: ORB thread : priority:5, demon:false, threadId:83, threadState:WAITING, lockName:java.lang.Object@6bf09f
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.orb.ORB.run(Unknown Source)
                 org.jboss.iiop.CorbaORBService$1.run(CorbaORBService.java:274)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RequestController-2 : priority:10, demon:true, threadId:84, threadState:WAITING, lockName:java.lang.Object@82602b
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: RequestController-3 : priority:10, demon:true, threadId:86, threadState:WAITING, lockName:java.lang.Object@177deb4
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: RequestController-4 : priority:10, demon:true, threadId:87, threadState:WAITING, lockName:java.lang.Object@be2e70
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: RequestController-5 : priority:10, demon:true, threadId:90, threadState:WAITING, lockName:java.lang.Object@1dae580
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: RequestController-6 : priority:10, demon:true, threadId:92, threadState:WAITING, lockName:java.lang.Object@11594d
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestController.waitForQueue(Unknown Source)
                 org.jacorb.poa.RequestController.run(Unknown Source)
                
                 Thread: ClientMessageReceptor0 : priority:5, demon:true, threadId:94, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 org.jacorb.orb.etf.StreamConnectionBase.read(Unknown Source)
                 org.jacorb.orb.giop.GIOPConnection.getMessage(Unknown Source)
                 org.jacorb.orb.giop.GIOPConnection.receiveMessages(Unknown Source)
                 org.jacorb.orb.giop.MessageReceptor.doWork(Unknown Source)
                 org.jacorb.util.threadpool.ConsumerTie.run(Unknown Source)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ServerMessageReceptor0 : priority:5, demon:true, threadId:95, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 org.jacorb.orb.etf.StreamConnectionBase.read(Unknown Source)
                 org.jacorb.orb.giop.GIOPConnection.getMessage(Unknown Source)
                 org.jacorb.orb.giop.GIOPConnection.receiveMessages(Unknown Source)
                 org.jacorb.orb.giop.MessageReceptor.doWork(Unknown Source)
                 org.jacorb.util.threadpool.ConsumerTie.run(Unknown Source)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: RequestProcessor-1 : priority:10, demon:true, threadId:96, threadState:WAITING, lockName:org.jacorb.poa.RequestProcessor@12798c1
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestProcessor.run(Unknown Source)
                
                 Thread: RequestProcessor-2 : priority:10, demon:true, threadId:97, threadState:WAITING, lockName:org.jacorb.poa.RequestProcessor@9dc852
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jacorb.poa.RequestProcessor.run(Unknown Source)
                
                 Thread: ContainerBackgroundProcessor[StandardEngine[jboss.web]] : priority:5, demon:true, threadId:98, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1578)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-22 : priority:5, demon:true, threadId:99, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 org.jboss.console.plugins.AOPLister$1.run(AOPLister.java:882)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: DefaultQuartzScheduler_Worker-0 : priority:5, demon:false, threadId:100, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-1 : priority:5, demon:false, threadId:101, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-2 : priority:5, demon:false, threadId:102, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-3 : priority:5, demon:false, threadId:103, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-4 : priority:5, demon:false, threadId:104, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-5 : priority:5, demon:false, threadId:105, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-6 : priority:5, demon:false, threadId:106, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-7 : priority:5, demon:false, threadId:107, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-8 : priority:5, demon:false, threadId:108, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: DefaultQuartzScheduler_Worker-9 : priority:5, demon:false, threadId:109, threadState:TIMED_WAITING, lockName:java.lang.Object@fdfb66
                
                 java.lang.Object.wait(Native Method)
                 org.quartz.simpl.SimpleThreadPool.getNextRunnable(SimpleThreadPool.java:428)
                 org.quartz.simpl.SimpleThreadPool.access$000(SimpleThreadPool.java:47)
                 org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:518)
                
                 Thread: ScannerThread : priority:5, demon:true, threadId:111, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.loop(AbstractDeploymentScanner.java:280)
                 org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.run(AbstractDeploymentScanner.java:225)
                
                 Thread: Timer-5 : priority:5, demon:true, threadId:123, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@841435
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: IdleRemover : priority:5, demon:true, threadId:124, threadState:TIMED_WAITING, lockName:java.util.ArrayList@14ad0f9
                
                 java.lang.Object.wait(Native Method)
                 org.jboss.resource.connectionmanager.IdleRemover$IdleRemoverRunnable.run(IdleRemover.java:161)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: JCA PoolFiller : priority:5, demon:false, threadId:125, threadState:WAITING, lockName:java.util.LinkedList@ddb000
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jboss.resource.connectionmanager.PoolFiller.run(PoolFiller.java:87)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-34 : priority:5, demon:true, threadId:126, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 org.jboss.messaging.core.impl.memory.SimpleMemoryManager$MemoryRunnable.run(SimpleMemoryManager.java:126)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Timer-6 : priority:5, demon:true, threadId:127, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@4dd071
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: Timer-7 : priority:5, demon:true, threadId:129, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@afd908
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: Scheduler main thread : priority:5, demon:true, threadId:133, threadState:WAITING, lockName:java.lang.Object@d1b21c
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jgroups.util.Queue.peek(Queue.java:404)
                 org.jgroups.util.Scheduler.run(Scheduler.java:77)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Timer-8 : priority:5, demon:true, threadId:160, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@1048d4f
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: Timer-9 : priority:5, demon:true, threadId:161, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@1d97b9b
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: control: Socket[addr=app1.exp.elynx.net/10.33.32.21,port=2109,localport=32808] : priority:5, demon:true, threadId:162, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.net.SocketInputStream.read(SocketInputStream.java:182)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$ControlConnectionThread.run(BisocketServerInvoker.java:741)
                
                 Thread: Timer-10 : priority:5, demon:true, threadId:163, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@1420a5f
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: WorkerThread#0[10.33.32.21:32850] : priority:5, demon:false, threadId:164, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
                 java.io.BufferedInputStream.read(BufferedInputStream.java:235)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.socket.ServerThread.readVersion(ServerThread.java:824)
                 org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:510)
                 org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
                 org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
                
                 Thread: Timer-11 : priority:5, demon:true, threadId:165, threadState:TIMED_WAITING, lockName:java.util.TaskQueue@b6ede0
                
                 java.lang.Object.wait(Native Method)
                 java.util.TimerThread.mainLoop(Timer.java:509)
                 java.util.TimerThread.run(Timer.java:462)
                
                 Thread: Thread-47 : priority:5, demon:false, threadId:166, threadState:WAITING, lockName:java.lang.Object@1bb1eee
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-48 : priority:5, demon:false, threadId:167, threadState:WAITING, lockName:java.lang.Object@1eba76d
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-49 : priority:5, demon:false, threadId:168, threadState:WAITING, lockName:java.lang.Object@3d25b0
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-50 : priority:5, demon:false, threadId:169, threadState:WAITING, lockName:java.lang.Object@758482
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-51 : priority:5, demon:false, threadId:170, threadState:WAITING, lockName:java.lang.Object@14e95ea
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-52 : priority:5, demon:false, threadId:171, threadState:WAITING, lockName:java.lang.Object@5476f2
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-53 : priority:5, demon:false, threadId:172, threadState:WAITING, lockName:java.lang.Object@14d0a09
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-54 : priority:5, demon:false, threadId:173, threadState:WAITING, lockName:java.lang.Object@cdea0e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-55 : priority:5, demon:false, threadId:174, threadState:WAITING, lockName:java.lang.Object@c13d8b
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-24 : priority:5, demon:false, threadId:114, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@d81341
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-23 : priority:5, demon:false, threadId:113, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@b5f87c
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-25 : priority:5, demon:false, threadId:115, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@1372bdb
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-26 : priority:5, demon:false, threadId:116, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@d6e0df
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: control: Socket[addr=app1.exp.elynx.net/10.33.32.21,port=2109,localport=32812] : priority:5, demon:true, threadId:180, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.net.SocketInputStream.read(SocketInputStream.java:182)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$ControlConnectionThread.run(BisocketServerInvoker.java:741)
                
                 Thread: Thread-57 : priority:5, demon:false, threadId:181, threadState:WAITING, lockName:java.lang.Object@af9911
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-58 : priority:5, demon:false, threadId:182, threadState:WAITING, lockName:java.lang.Object@6a64e3
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-59 : priority:5, demon:false, threadId:183, threadState:WAITING, lockName:java.lang.Object@1057f9a
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-60 : priority:5, demon:false, threadId:184, threadState:WAITING, lockName:java.lang.Object@1fc916e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-61 : priority:5, demon:false, threadId:185, threadState:WAITING, lockName:java.lang.Object@afdd29
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-62 : priority:5, demon:false, threadId:186, threadState:WAITING, lockName:java.lang.Object@1e0efb9
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-63 : priority:5, demon:false, threadId:187, threadState:WAITING, lockName:java.lang.Object@9a819a
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-64 : priority:5, demon:false, threadId:188, threadState:WAITING, lockName:java.lang.Object@1b2b421
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-66 : priority:5, demon:false, threadId:189, threadState:WAITING, lockName:java.lang.Object@15015a9
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-65 : priority:5, demon:false, threadId:190, threadState:WAITING, lockName:java.lang.Object@be4043
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-67 : priority:5, demon:false, threadId:191, threadState:WAITING, lockName:java.lang.Object@6cf331
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-68 : priority:5, demon:false, threadId:192, threadState:WAITING, lockName:java.lang.Object@d21735
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-69 : priority:5, demon:false, threadId:193, threadState:WAITING, lockName:java.lang.Object@f1b8cb
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-70 : priority:5, demon:false, threadId:194, threadState:WAITING, lockName:java.lang.Object@19b41f7
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-71 : priority:5, demon:false, threadId:195, threadState:WAITING, lockName:java.lang.Object@7fe87
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-27 : priority:5, demon:false, threadId:117, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@1a928f9
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-72 : priority:5, demon:false, threadId:196, threadState:WAITING, lockName:java.lang.Object@d75843
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-28 : priority:5, demon:false, threadId:118, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@7a1116
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: control: Socket[addr=app1.exp.elynx.net/10.33.32.21,port=2109,localport=32826] : priority:5, demon:true, threadId:197, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.net.SocketInputStream.read(SocketInputStream.java:182)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$ControlConnectionThread.run(BisocketServerInvoker.java:741)
                
                 Thread: Thread-29 : priority:5, demon:false, threadId:119, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@115ae0
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-74 : priority:5, demon:false, threadId:198, threadState:WAITING, lockName:java.lang.Object@1e3aebb
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-30 : priority:5, demon:false, threadId:120, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@551f11
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: control: Socket[addr=app1.exp.elynx.net/10.33.32.21,port=2109,localport=32831] : priority:5, demon:true, threadId:199, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.net.SocketInputStream.read(SocketInputStream.java:182)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$ControlConnectionThread.run(BisocketServerInvoker.java:741)
                
                 Thread: Thread-31 : priority:5, demon:false, threadId:121, threadState:WAITING, lockName:com.eLynx.Messaging.MessageReceiver@c0468e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 com.eLynx.Messaging.MessageReceiver.listen(MessageReceiver.java:432)
                 com.eLynx.Messaging.MessageReceiver.run(MessageReceiver.java:835)
                
                 Thread: Thread-76 : priority:5, demon:false, threadId:200, threadState:WAITING, lockName:java.lang.Object@e495b4
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: JBossLifeThread : priority:5, demon:false, threadId:201, threadState:WAITING, lockName:java.lang.Object@1208e1e
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jboss.system.server.ServerImpl$LifeThread.run(ServerImpl.java:940)
                
                 Thread: http-0.0.0.0-8080-Poller-0 : priority:5, demon:true, threadId:202, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Poller@1f2fbf5
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Poller.run(AprEndpoint.java:1266)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: http-0.0.0.0-8080-CometPoller-0 : priority:5, demon:true, threadId:203, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Poller@5273f2
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Poller.run(AprEndpoint.java:1266)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: http-0.0.0.0-8080-Sendfile-0 : priority:5, demon:true, threadId:204, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Sendfile@13a4f81
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Sendfile.run(AprEndpoint.java:1721)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: http-0.0.0.0-8080-Acceptor-0 : priority:5, demon:true, threadId:205, threadState:RUNNABLE, lockName:null
                
                 org.apache.tomcat.jni.Socket.accept(Native Method)
                 org.apache.tomcat.util.net.AprEndpoint$Acceptor.run(AprEndpoint.java:1109)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-Poller-0 : priority:5, demon:true, threadId:206, threadState:RUNNABLE, lockName:null
                
                 org.apache.tomcat.jni.Poll.poll(Native Method)
                 org.apache.tomcat.util.net.AprEndpoint$Poller.run(AprEndpoint.java:1297)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-CometPoller-0 : priority:5, demon:true, threadId:207, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Poller@1399fce
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Poller.run(AprEndpoint.java:1266)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-Acceptor-0 : priority:5, demon:true, threadId:208, threadState:RUNNABLE, lockName:null
                
                 org.apache.tomcat.jni.Socket.accept(Native Method)
                 org.apache.tomcat.util.net.AprEndpoint$Acceptor.run(AprEndpoint.java:1109)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-1 : priority:5, demon:true, threadId:209, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Worker@2ac35a
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.await(AprEndpoint.java:1464)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1489)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-2 : priority:5, demon:true, threadId:210, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Worker@17f6db3
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.await(AprEndpoint.java:1464)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1489)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-3 : priority:5, demon:true, threadId:211, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Worker@fb13f3
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.await(AprEndpoint.java:1464)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1489)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-6 : priority:5, demon:false, threadId:28, threadState:TIMED_WAITING, lockName:null
                
                 java.lang.Thread.sleep(Native Method)
                 com.arjuna.ats.internal.arjuna.recovery.PeriodicRecovery.doWork(PeriodicRecovery.java:248)
                 com.arjuna.ats.internal.arjuna.recovery.PeriodicRecovery.run(PeriodicRecovery.java:163)
                
                 Thread: ajp-0.0.0.0-8009-4 : priority:5, demon:true, threadId:212, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Worker@1dad1b1
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.await(AprEndpoint.java:1464)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1489)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ajp-0.0.0.0-8009-5 : priority:5, demon:true, threadId:213, threadState:WAITING, lockName:org.apache.tomcat.util.net.AprEndpoint$Worker@fce46c
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.await(AprEndpoint.java:1464)
                 org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1489)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: ReusableThread : priority:5, demon:true, threadId:221, threadState:WAITING, lockName:org.jgroups.util.ReusableThread@f88273
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 org.jgroups.util.ReusableThread.run(ReusableThread.java:201)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: WorkerThread#1[10.33.32.23:32810] : priority:5, demon:false, threadId:224, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
                 java.io.BufferedInputStream.read(BufferedInputStream.java:235)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.socket.ServerThread.readVersion(ServerThread.java:824)
                 org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:510)
                 org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
                 org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
                
                 Thread: control: Socket[addr=app3.exp.elynx.net/10.33.32.23,port=2246,localport=32843] : priority:5, demon:true, threadId:225, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.net.SocketInputStream.read(SocketInputStream.java:182)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.bisocket.BisocketServerInvoker$ControlConnectionThread.run(BisocketServerInvoker.java:741)
                
                 Thread: Thread-87 : priority:5, demon:false, threadId:226, threadState:WAITING, lockName:java.lang.Object@13c5efb
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-88 : priority:5, demon:false, threadId:227, threadState:WAITING, lockName:java.lang.Object@7fcd11
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-89 : priority:5, demon:false, threadId:228, threadState:WAITING, lockName:java.lang.Object@704688
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-90 : priority:5, demon:false, threadId:229, threadState:WAITING, lockName:java.lang.Object@126b0e6
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-91 : priority:5, demon:false, threadId:230, threadState:WAITING, lockName:java.lang.Object@127d4d6
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-92 : priority:5, demon:false, threadId:231, threadState:WAITING, lockName:java.lang.Object@62f38
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-93 : priority:5, demon:false, threadId:232, threadState:WAITING, lockName:java.lang.Object@61c1c
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: Thread-94 : priority:5, demon:false, threadId:233, threadState:WAITING, lockName:java.lang.Object@11f983b
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)
                 java.lang.Thread.run(Thread.java:595)
                
                 Thread: WorkerThread#2[10.33.32.23:32836] : priority:5, demon:false, threadId:234, threadState:RUNNABLE, lockName:null
                
                 java.net.SocketInputStream.socketRead0(Native Method)
                 java.net.SocketInputStream.read(SocketInputStream.java:129)
                 java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
                 java.io.BufferedInputStream.read(BufferedInputStream.java:235)
                 java.io.FilterInputStream.read(FilterInputStream.java:66)
                 org.jboss.remoting.transport.socket.ServerThread.readVersion(ServerThread.java:824)
                 org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:510)
                 org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
                 org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
                
                 Thread: Thread-96 : priority:5, demon:false, threadId:235, threadState:WAITING, lockName:java.lang.Object@143762c
                
                 java.lang.Object.wait(Native Method)
                 java.lang.Object.wait(Object.java:474)
                 EDU.oswego.cs.dl.util.concurrent.LinkedQueue.take(LinkedQueue.java:122)
                 EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:83)