root@ip-10-4-3-22:/usr/lib/jvm/java-8-oracle/bin# ./jstack -l 16046 2019-07-15 16:14:32 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.151-b12 mixed mode): "Attach Listener" #1009 daemon prio=9 os_prio=0 tid=0x00007f5d10001800 nid=0x2a93 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "ForkJoinPool-1-worker-9" #1005 daemon prio=5 os_prio=0 tid=0x00007f5d38038800 nid=0x2273 waiting on condition [0x00007f5c34f84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8e738> (a scala.concurrent.forkjoin.ForkJoinPool) at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-26" #468 prio=5 os_prio=0 tid=0x00007f5c40118000 nid=0x7cb0 waiting on condition [0x00007f5cfd2b0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-24" #470 prio=5 os_prio=0 tid=0x00007f5c44bf2000 nid=0x7caf waiting on condition [0x00007f5c3598a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-25" #469 prio=5 os_prio=0 tid=0x00007f5c4c0f4000 nid=0x7cae waiting on condition [0x00007f5c35c8d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-22" #466 prio=5 os_prio=0 tid=0x00007f5d3c02f800 nid=0x7bdd waiting on condition [0x00007f5c35d8e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-23" #467 prio=5 os_prio=0 tid=0x00007f5d40033000 nid=0x7bdc waiting on condition [0x00007f5c35b8c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-21" #465 prio=5 os_prio=0 tid=0x00007f5c40104800 nid=0x7bdb waiting on condition [0x00007f5c35a8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502_ip-10-4-3-22-1562699646519-781ae911-0-3" #351 prio=5 os_prio=0 tid=0x00007f5cb8088800 nid=0x1978 runnable [0x00007f5c37e9b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e5cc5480> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e5cc54b0> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e5cc5490> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e5cc5530> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e5cc55b8> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-20" #182 prio=5 os_prio=0 tid=0x00007f5cf8040800 nid=0x40b4 waiting on condition [0x00007f5c35e8f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-19" #181 prio=5 os_prio=0 tid=0x00007f5cf4024800 nid=0x40b3 waiting on condition [0x00007f5c35f90000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-18" #180 prio=5 os_prio=0 tid=0x00007f5c40106000 nid=0x40b2 waiting on condition [0x00007f5c36091000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-16" #178 prio=5 os_prio=0 tid=0x00007f5d4400d000 nid=0x40b1 waiting on condition [0x00007f5cfcdab000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-15" #177 prio=5 os_prio=0 tid=0x00007f5d2013f800 nid=0x40b0 waiting on condition [0x00007f5cfc9a7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-14" #176 prio=5 os_prio=0 tid=0x00007f5d3c024000 nid=0x40af waiting on condition [0x00007f5cfd1af000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-17" #179 prio=5 os_prio=0 tid=0x00007f5d3801f000 nid=0x40ae waiting on condition [0x00007f5cfceac000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-13" #175 prio=5 os_prio=0 tid=0x00007f5d4002a000 nid=0x40ad waiting on condition [0x00007f5caf759000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "New Relic Segment Expiration Handler 2" #170 daemon prio=5 os_prio=0 tid=0x00007f5c70001800 nid=0x408a waiting on condition [0x00007f5cfcaa8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0731590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-12" #169 prio=5 os_prio=0 tid=0x00007f5d3801c000 nid=0x4044 waiting on condition [0x00007f5cfccaa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-9" #165 prio=5 os_prio=0 tid=0x00007f5d14519800 nid=0x4042 waiting on condition [0x00007f5cfcfad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-11" #168 prio=5 os_prio=0 tid=0x00007f5d040f9000 nid=0x4040 waiting on condition [0x00007f5cfd0ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-10" #167 prio=5 os_prio=0 tid=0x00007f5cf4025800 nid=0x403f waiting on condition [0x00007f5c76ff2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-8" #166 prio=5 os_prio=0 tid=0x00007f5cf8041800 nid=0x403e waiting on condition [0x00007f5caeffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "New Relic Segment Expiration Handler" #162 daemon prio=5 os_prio=0 tid=0x00007f5cf00b5800 nid=0x3f99 waiting on condition [0x00007f5cac737000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0731590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-7" #161 prio=5 os_prio=0 tid=0x00007f5cf4012000 nid=0x3f6b waiting on condition [0x00007f5c36792000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "New Relic Harvest Service" #160 daemon prio=5 os_prio=0 tid=0x00007f5d040ff800 nid=0x3f68 waiting on condition [0x00007f5c36893000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0cde580> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Token Expiration Handler" #159 daemon prio=5 os_prio=0 tid=0x00007f5d04072800 nid=0x3f67 waiting on condition [0x00007f5caca38000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0731328> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.io.pinned-dispatcher-6" #158 prio=5 os_prio=0 tid=0x00007f5d04044000 nid=0x3f66 runnable [0x00007f5c36b94000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfdf0160> (a sun.nio.ch.Util$3) - locked <0x00000005dfdf0150> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd84b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at akka.io.SelectionHandler$ChannelRegistryImpl$$anon$3.tryRun(SelectionHandler.scala:146) at akka.io.SelectionHandler$ChannelRegistryImpl$Task.run(SelectionHandler.scala:286) at akka.io.SelectionHandler$ChannelRegistryImpl$$anon$3.run(SelectionHandler.scala:179) at akka.util.SerializedSuspendableExecutionContext.run$1(SerializedSuspendableExecutionContext.scala:72) at akka.util.SerializedSuspendableExecutionContext.run(SerializedSuspendableExecutionContext.scala:77) at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd84450> (a java.util.concurrent.ThreadPoolExecutor$Worker) "bundle-188-ActorSystem-akka.actor.default-dispatcher-5" #157 prio=5 os_prio=0 tid=0x00007f5c449cd000 nid=0x3f65 waiting on condition [0x00007f5c37495000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-4" #156 prio=5 os_prio=0 tid=0x00007f5cf0051800 nid=0x3f64 waiting on condition [0x00007f5c37596000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-3" #155 prio=5 os_prio=0 tid=0x00007f5cf4011000 nid=0x3f63 waiting on condition [0x00007f5c37697000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-akka.actor.default-dispatcher-2" #154 prio=5 os_prio=0 tid=0x00007f5c449bb000 nid=0x3f62 waiting on condition [0x00007f5c37798000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfa8f008> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-188-ActorSystem-scheduler-1" #153 prio=5 os_prio=0 tid=0x00007f5c44939000 nid=0x3f61 sleeping[0x00007f5c37a99000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at akka.actor.LightArrayRevolverScheduler.waitNanos(LightArrayRevolverScheduler.scala:87) at akka.actor.LightArrayRevolverScheduler$$anon$4.nextTick(LightArrayRevolverScheduler.scala:271) at akka.actor.LightArrayRevolverScheduler$$anon$4.run(LightArrayRevolverScheduler.scala:241) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502_ip-10-4-3-22-1562699646519-781ae911-0-1" #152 prio=5 os_prio=0 tid=0x00007f5cb808a800 nid=0x3f60 runnable [0x00007f5c37d9a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e01b1fc0> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e01b1f90> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e01b2148> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e01b8570> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e01b84c8> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502_ip-10-4-3-22-1562699646519-781ae911-0-2" #150 prio=5 os_prio=0 tid=0x00007f5cb8071000 nid=0x3f5e runnable [0x00007f5c37f9c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e01add18> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e01adce8> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e01ade40> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e01ae218> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e01ae170> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #149 prio=5 os_prio=0 tid=0x00007f5c44687000 nid=0x3f5d waiting on condition [0x00007f5c3829d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0143c80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0143e10> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #148 prio=5 os_prio=0 tid=0x00007f5c44685000 nid=0x3f5c waiting on condition [0x00007f5c3839e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0144aa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e01447a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #147 prio=5 os_prio=0 tid=0x00007f5c44683000 nid=0x3f5b waiting on condition [0x00007f5c3849f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0144738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0144430> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #146 prio=5 os_prio=0 tid=0x00007f5c44681800 nid=0x3f5a waiting on condition [0x00007f5c385a0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0146db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0146ab0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #145 prio=5 os_prio=0 tid=0x00007f5c4467f800 nid=0x3f59 waiting on condition [0x00007f5c386a1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01443c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0144068> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #144 prio=5 os_prio=0 tid=0x00007f5c4467e000 nid=0x3f58 waiting on condition [0x00007f5c387a2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0146a48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0146548> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #143 prio=5 os_prio=0 tid=0x00007f5c4467c800 nid=0x3f57 waiting on condition [0x00007f5c388a3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01464c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0144b10> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502-0" #142 prio=5 os_prio=0 tid=0x00007f5c4467a800 nid=0x3f56 waiting on condition [0x00007f5c389a4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01475c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0146e20> (a java.util.concurrent.ThreadPoolExecutor$Worker) "uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502_ip-10-4-3-22-1562699646519-781ae911-leader-finder-thread" #141 prio=5 os_prio=0 tid=0x00007f5c44620800 nid=0x3f55 waiting on condition [0x00007f5c38aa5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01cee80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at kafka.consumer.ConsumerFetcherManager$LeaderFinderThread.doWork(ConsumerFetcherManager.scala:61) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "uriCacheInvalidator-ip-10-4-3-229980b083-0cf7-4d84-8702-4ac9816cb502_ip-10-4-3-22-1562699646519-781ae911_watcher_executor" #140 prio=5 os_prio=0 tid=0x00007f5c444fd800 nid=0x3f54 runnable [0x00007f5c38ba6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01c2ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.consumer.ZookeeperConsumerConnector$ZKRebalancerListener$$anon$1.run(ZookeeperConsumerConnector.scala:333) Locked ownable synchronizers: - None "pool-10-thread-1-EventThread" #139 daemon prio=5 os_prio=0 tid=0x00007f5c444c6000 nid=0x3f53 waiting on condition [0x00007f5c38ca7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01ba188> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-10-thread-1-SendThread(10.4.3.136:2181)" #138 daemon prio=5 os_prio=0 tid=0x00007f5c444c4800 nid=0x3f52 runnable [0x00007f5c38da8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e01b9b38> (a sun.nio.ch.Util$3) - locked <0x00000005e01b9b28> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e01b96c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ZkClient-EventThread-137-10.4.3.136:2181,10.4.2.59:2181,10.4.3.116:2181" #137 daemon prio=5 os_prio=0 tid=0x00007f5c444c8000 nid=0x3f51 waiting on condition [0x00007f5c394f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e01cea28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) Locked ownable synchronizers: - None "I/O dispatcher 8" #136 prio=5 os_prio=0 tid=0x00007f5c80108800 nid=0x3f50 runnable [0x00007f5c399f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff36078> (a sun.nio.ch.Util$3) - locked <0x00000005dff36068> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff35f30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 7" #135 prio=5 os_prio=0 tid=0x00007f5c80106800 nid=0x3f4f runnable [0x00007f5c39af3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff36550> (a sun.nio.ch.Util$3) - locked <0x00000005dff36540> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff36408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 6" #134 prio=5 os_prio=0 tid=0x00007f5c80104800 nid=0x3f4e runnable [0x00007f5c39bf4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff3ea90> (a sun.nio.ch.Util$3) - locked <0x00000005dff3ea80> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff3e948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 5" #133 prio=5 os_prio=0 tid=0x00007f5c80102800 nid=0x3f4d runnable [0x00007f5c39cf5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff3ff78> (a sun.nio.ch.Util$3) - locked <0x00000005dff3ff68> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff3fe30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 4" #132 prio=5 os_prio=0 tid=0x00007f5c80100800 nid=0x3f4c runnable [0x00007f5c39df6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff40450> (a sun.nio.ch.Util$3) - locked <0x00000005dff40440> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff40308> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 3" #131 prio=5 os_prio=0 tid=0x00007f5c800fe800 nid=0x3f4b runnable [0x00007f5c39ef7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff0a328> (a sun.nio.ch.Util$3) - locked <0x00000005dff0a318> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff0a1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 2" #130 prio=5 os_prio=0 tid=0x00007f5c80082000 nid=0x3f4a runnable [0x00007f5c39ff8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff25b40> (a sun.nio.ch.Util$3) - locked <0x00000005dff25b30> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff259f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 1" #129 prio=5 os_prio=0 tid=0x00007f5c8007f000 nid=0x3f49 runnable [0x00007f5c741df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff272e8> (a sun.nio.ch.Util$3) - locked <0x00000005dff272d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff271a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-18-thread-1" #128 prio=5 os_prio=0 tid=0x00007f5c442d8800 nid=0x3f48 runnable [0x00007f5c742e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dff34c78> (a sun.nio.ch.Util$3) - locked <0x00000005dff34c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dff34b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:340) at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221) at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-6" #114 prio=5 os_prio=0 tid=0x00007f5c4424e000 nid=0x3f47 runnable [0x00007f5c745e1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da439e48> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da43bec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da439db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-5" #113 prio=5 os_prio=0 tid=0x00007f5c4424c000 nid=0x3f46 runnable [0x00007f5c746e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da43d770> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da43f7f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da43d6d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-4" #112 prio=5 os_prio=0 tid=0x00007f5c4424a800 nid=0x3f45 runnable [0x00007f5c747e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da441098> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da443118> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da441000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-3" #111 prio=5 os_prio=0 tid=0x00007f5c44248800 nid=0x3f44 runnable [0x00007f5c748e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da4449c0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da446a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da444928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-2" #110 prio=5 os_prio=0 tid=0x00007f5c44247000 nid=0x3f43 runnable [0x00007f5c749e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da4482e8> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da44a368> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da448250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-1" #109 prio=5 os_prio=0 tid=0x00007f5c44244800 nid=0x3f42 runnable [0x00007f5c74ae6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da44bc10> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da44dc90> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da44bb78> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "threadDeathWatcher-2-1" #127 daemon prio=1 os_prio=0 tid=0x00007f5c5809d800 nid=0x3f3e waiting on condition [0x00007f5c75be8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:137) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-timeouter-0" #124 prio=5 os_prio=0 tid=0x00007f5c58021800 nid=0x3f3d sleeping[0x00007f5c75ce9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:461) at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:360) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-0" #108 prio=5 os_prio=0 tid=0x00007f5c441d0000 nid=0x3f27 runnable [0x00007f5c75fea000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005da44f538> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005da4515b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005da44f4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-scheduled-task-worker-0" #125 prio=5 os_prio=0 tid=0x00007f5c4418f800 nid=0x3f25 waiting on condition [0x00007f5c760eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005da452dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-connection-reaper-0" #107 prio=5 os_prio=0 tid=0x00007f5c4406c000 nid=0x3f24 waiting on condition [0x00007f5c761ec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8c3c808> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-8" #106 prio=5 os_prio=0 tid=0x00007f5c44009000 nid=0x3f23 waiting on condition [0x00007f5c76aed000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-7" #105 prio=5 os_prio=0 tid=0x00007f5c44007000 nid=0x3f22 waiting on condition [0x00007f5c76bee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-6" #104 prio=5 os_prio=0 tid=0x00007f5c44005000 nid=0x3f21 waiting on condition [0x00007f5c76cef000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-5" #103 prio=5 os_prio=0 tid=0x00007f5c44003000 nid=0x3f20 waiting on condition [0x00007f5c76df0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-4" #102 prio=5 os_prio=0 tid=0x00007f5c44001000 nid=0x3f1f waiting on condition [0x00007f5c76ef1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "CuratorFramework-0" #100 daemon prio=5 os_prio=0 tid=0x00007f5c8c290000 nid=0x3f1d waiting on condition [0x00007f5c770f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8dd2938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.DelayQueue.take(DelayQueue.java:211) at java.util.concurrent.DelayQueue.take(DelayQueue.java:70) at com.netflix.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:632) at com.netflix.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:53) at com.netflix.curator.framework.imps.CuratorFrameworkImpl$3.call(CuratorFrameworkImpl.java:241) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d8dd2b38> (a java.util.concurrent.ThreadPoolExecutor$Worker) "pool-7-thread-1-EventThread" #99 daemon prio=5 os_prio=0 tid=0x00007f5c8c28e000 nid=0x3f1c waiting on condition [0x00007f5c771f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8dd21d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-7-thread-1-SendThread(10.4.3.136:2181)" #98 daemon prio=5 os_prio=0 tid=0x00007f5c8c28d000 nid=0x3f1b runnable [0x00007f5c772f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d8dd07b8> (a sun.nio.ch.Util$3) - locked <0x00000005d8dd07a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d8dd0340> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ConnectionStateManager-0" #97 daemon prio=5 os_prio=0 tid=0x00007f5c8c27e000 nid=0x3f1a waiting on condition [0x00007f5c773f6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8dd3090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at com.netflix.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:170) at com.netflix.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:40) at com.netflix.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:104) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d8dd3298> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ProducerSendThread-" #96 prio=5 os_prio=0 tid=0x00007f5c8c266000 nid=0x3f19 runnable [0x00007f5c774f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8ddc4e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at scala.collection.immutable.Stream$.continually(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x000000072453b2b8> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x000000072453b2e8> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream.foreach(Stream.scala:595) at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:66) at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:44) Locked ownable synchronizers: - None "ProducerSendThread-" #95 prio=5 os_prio=0 tid=0x00007f5c8c264000 nid=0x3f18 runnable [0x00007f5c775f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8ddb448> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at scala.collection.immutable.Stream$.continually(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007244b3358> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007244b3388> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream.foreach(Stream.scala:595) at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:66) at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:44) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #94 daemon prio=5 os_prio=0 tid=0x00007f5c8c25c800 nid=0x3f17 waiting on condition [0x00007f5c776f9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8ddab98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #93 daemon prio=5 os_prio=0 tid=0x00007f5c8c06d000 nid=0x3f16 waiting on condition [0x00007f5c777fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8ddab98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-7-thread-1-EventThread" #91 daemon prio=5 os_prio=0 tid=0x00007f5c8c02c800 nid=0x3f15 waiting on condition [0x00007f5c77afb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8dd7dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-7-thread-1-SendThread(10.4.2.59:2181)" #90 daemon prio=5 os_prio=0 tid=0x00007f5c8c01d800 nid=0x3f14 runnable [0x00007f5c77bfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d8dd6580> (a sun.nio.ch.Util$3) - locked <0x00000005d8dd6570> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d8dd6108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ZkClient-EventThread-89-10.4.3.136:2181,10.4.2.59:2181,10.4.3.116:2181" #89 daemon prio=5 os_prio=0 tid=0x00007f5c8c01c000 nid=0x3f13 waiting on condition [0x00007f5c77cfd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8dd8690> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) Locked ownable synchronizers: - None "LocatorPool-3-EventThread" #88 daemon prio=5 os_prio=0 tid=0x00007f5c78024000 nid=0x3f12 waiting on condition [0x00007f5c77ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d7967b48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "LocatorPool-3-SendThread(10.4.2.59:2181)" #87 daemon prio=5 os_prio=0 tid=0x00007f5c78023000 nid=0x3f11 runnable [0x00007f5cac10d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d7969a90> (a sun.nio.ch.Util$3) - locked <0x00000005d7969a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d7969608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "LocatorPool-3" #86 prio=5 os_prio=0 tid=0x00007f5c8c008800 nid=0x3f10 waiting on condition [0x00007f5cac20e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-2" #84 prio=5 os_prio=0 tid=0x00007f5c8c00e000 nid=0x3f0f waiting on condition [0x00007f5cac535000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-1" #83 prio=5 os_prio=0 tid=0x00007f5c8c00c800 nid=0x3f0e waiting on condition [0x00007f5cac636000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d796b450> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixStartLevel" #80 daemon prio=5 os_prio=0 tid=0x00007f5cbc02e000 nid=0x3f0b in Object.wait() [0x00007f5cadaee000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkStartLevelImpl.run(FrameworkStartLevelImpl.java:283) - locked <0x00000005da473f60> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixFrameworkWiring" #79 daemon prio=5 os_prio=0 tid=0x00007f5cbc02b800 nid=0x3f0a in Object.wait() [0x00007f5cadbef000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005d967f178> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkWiringImpl.run(FrameworkWiringImpl.java:172) - locked <0x00000005d967f178> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixStartLevel" #77 daemon prio=5 os_prio=0 tid=0x00007f5cb41ea000 nid=0x3f08 in Object.wait() [0x00007f5caf557000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005d3f90af8> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkStartLevelImpl.run(FrameworkStartLevelImpl.java:283) - locked <0x00000005d3f90af8> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixFrameworkWiring" #76 daemon prio=5 os_prio=0 tid=0x00007f5cb41ef000 nid=0x3f07 in Object.wait() [0x00007f5caf658000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005d3f90728> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkWiringImpl.run(FrameworkWiringImpl.java:172) - locked <0x00000005d3f90728> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixDispatchQueue" #74 prio=5 os_prio=0 tid=0x00007f5cb417d000 nid=0x3f05 in Object.wait() [0x00007f5caf85a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.util.EventDispatcher.run(EventDispatcher.java:1118) - locked <0x00000005d6f057e0> (a java.util.ArrayList) at org.apache.felix.framework.util.EventDispatcher.access$000(EventDispatcher.java:55) at org.apache.felix.framework.util.EventDispatcher$1.run(EventDispatcher.java:102) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-5-thread-10" #73 prio=5 os_prio=0 tid=0x00007f5cd407f800 nid=0x3f04 in Object.wait() [0x00007f5cfcba9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.felix.framework.util.ThreadGate.await(ThreadGate.java:79) - locked <0x00000005d967f4d0> (a org.apache.felix.framework.util.ThreadGate) at org.apache.felix.framework.Felix.waitForStop(Felix.java:1068) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26$$anonfun$apply$30.apply(OsgiTasks.scala:284) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26$$anonfun$apply$30.apply(OsgiTasks.scala:284) at osgifelix.FelixRunner$.embed(FelixRunner.scala:60) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26.apply(OsgiTasks.scala:282) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26.apply(OsgiTasks.scala:273) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:47) at sbt.$tilde$greater$$anonfun$$u2219$1.apply(TypeFunctions.scala:40) at sbt.std.Transform$$anon$4.work(System.scala:63) at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:228) at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:228) at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:17) at sbt.Execute.work(Execute.scala:237) at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:228) at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:228) at sbt.ConcurrentRestrictions$$anon$4$$anonfun$1.apply(ConcurrentRestrictions.scala:159) at sbt.CompletionService$$anon$2.call(CompletionService.scala:28) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d6ebea78> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Service Thread" #34 daemon prio=9 os_prio=0 tid=0x00007f5d6c99f800 nid=0x3ed7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #33 daemon prio=9 os_prio=0 tid=0x00007f5d6c99d800 nid=0x3ed6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #32 daemon prio=9 os_prio=0 tid=0x00007f5d6c99b800 nid=0x3ed5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #31 daemon prio=9 os_prio=0 tid=0x00007f5d6c999800 nid=0x3ed4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #30 daemon prio=9 os_prio=0 tid=0x00007f5d6c999000 nid=0x3ed3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "New Relic RPM Connection Service" #29 daemon prio=5 os_prio=0 tid=0x00007f5d6c996000 nid=0x3ed2 waiting on condition [0x00007f5d24127000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c072de38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Deadlock Detector" #28 daemon prio=5 os_prio=0 tid=0x00007f5d6c991000 nid=0x3ed1 waiting on condition [0x00007f5cffdfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0aee408> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Sampler Service" #27 daemon prio=5 os_prio=0 tid=0x00007f5d6c98b800 nid=0x3ed0 waiting on condition [0x00007f5d2442a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0aef968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Transaction Service Processor" #26 daemon prio=5 os_prio=0 tid=0x00007f5d6c888000 nid=0x3ecf waiting on condition [0x00007f5cffffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c000bda8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Utilization Service 2" #25 daemon prio=5 os_prio=0 tid=0x00007f5d6c886000 nid=0x3ece waiting on condition [0x00007f5cffefd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0017f48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Utilization Service" #24 daemon prio=5 os_prio=0 tid=0x00007f5d6c885000 nid=0x3ecd waiting on condition [0x00007f5d24329000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0017f48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Retransformer" #23 daemon prio=5 os_prio=0 tid=0x00007f5d6c867000 nid=0x3ecc waiting on condition [0x00007f5d24228000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c001fd40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f5d6c207000 nid=0x3ebb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f5d6c1d4000 nid=0x3eba in Object.wait() [0x00007f5d25de4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000005c0000730> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f5d6c1cf800 nid=0x3eb9 in Object.wait() [0x00007f5d25ee5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000005c0000720> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f5d6c00a800 nid=0x3eaf waiting on condition [0x00007f5d72f33000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d6f11588> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:193) at sbt.ConcurrentRestrictions$$anon$4.take(ConcurrentRestrictions.scala:188) at sbt.Execute.next$1(Execute.scala:85) at sbt.Execute.processAll(Execute.scala:88) at sbt.Execute.runKeep(Execute.scala:68) at sbt.EvaluateTask$.liftedTree1$1(EvaluateTask.scala:359) at sbt.EvaluateTask$.run$1(EvaluateTask.scala:358) at sbt.EvaluateTask$.runTask(EvaluateTask.scala:378) at sbt.Aggregation$$anonfun$3.apply(Aggregation.scala:64) at sbt.Aggregation$$anonfun$3.apply(Aggregation.scala:62) at sbt.EvaluateTask$.withStreams(EvaluateTask.scala:314) at sbt.Aggregation$.timedRun(Aggregation.scala:62) at sbt.Aggregation$.runTasks(Aggregation.scala:71) at sbt.Aggregation$$anonfun$applyDynamicTasks$1.apply(Aggregation.scala:112) at sbt.Aggregation$$anonfun$applyDynamicTasks$1.apply(Aggregation.scala:110) at sbt.Command$$anonfun$applyEffect$2$$anonfun$apply$3.apply(Command.scala:61) at sbt.Command$$anonfun$applyEffect$2$$anonfun$apply$3.apply(Command.scala:61) at sbt.Act$$anonfun$sbt$Act$$actParser0$1$$anonfun$sbt$Act$$anonfun$$evaluate$1$1$$anonfun$apply$10.apply(Act.scala:244) at sbt.Act$$anonfun$sbt$Act$$actParser0$1$$anonfun$sbt$Act$$anonfun$$evaluate$1$1$$anonfun$apply$10.apply(Act.scala:241) at sbt.Command$.process(Command.scala:93) at sbt.MainLoop$$anonfun$1$$anonfun$apply$1.apply(MainLoop.scala:96) at sbt.MainLoop$$anonfun$1$$anonfun$apply$1.apply(MainLoop.scala:96) at sbt.State$$anon$1.process(State.scala:184) at sbt.MainLoop$$anonfun$1.apply(MainLoop.scala:96) at sbt.MainLoop$$anonfun$1.apply(MainLoop.scala:96) at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:17) at sbt.MainLoop$.next(MainLoop.scala:96) at sbt.MainLoop$.run(MainLoop.scala:89) at sbt.MainLoop$$anonfun$runWithNewLog$1.apply(MainLoop.scala:68) at sbt.MainLoop$$anonfun$runWithNewLog$1.apply(MainLoop.scala:63) at sbt.Using.apply(Using.scala:24) at sbt.MainLoop$.runWithNewLog(MainLoop.scala:63) at sbt.MainLoop$.runAndClearLast(MainLoop.scala:46) at sbt.MainLoop$.runLoggedLoop(MainLoop.scala:30) at sbt.MainLoop$.runLogged(MainLoop.scala:22) at sbt.StandardMain$.runManaged(Main.scala:54) at sbt.xMain.run(Main.scala:29) at xsbt.boot.Launch$$anonfun$run$1.apply(Launch.scala:109) at xsbt.boot.Launch$.withContextLoader(Launch.scala:128) at xsbt.boot.Launch$.run(Launch.scala:109) at xsbt.boot.Launch$$anonfun$apply$1.apply(Launch.scala:35) at xsbt.boot.Launch$.launch(Launch.scala:117) at xsbt.boot.Launch$.apply(Launch.scala:18) at xsbt.boot.Boot$.runImpl(Boot.scala:41) at xsbt.boot.Boot$.main(Boot.scala:17) at xsbt.boot.Boot.main(Boot.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f5d6c1c8000 nid=0x3eb8 runnable "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f5d6c020000 nid=0x3eb0 runnable "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f5d6c021800 nid=0x3eb1 runnable "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f5d6c023800 nid=0x3eb2 runnable "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f5d6c025000 nid=0x3eb3 runnable "GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f5d6c027000 nid=0x3eb4 runnable "GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f5d6c028800 nid=0x3eb5 runnable "GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f5d6c02a800 nid=0x3eb6 runnable "GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f5d6c02c000 nid=0x3eb7 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f5d6c975800 nid=0x3ed8 waiting on condition JNI global references: 399