root@ip-10-2-2-20:/usr/lib/jvm/java-8-oracle/bin# ./jstack -l 1365 2019-07-16 19:40:37 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.151-b12 mixed mode): "Attach Listener" #114967 daemon prio=9 os_prio=0 tid=0x00007f8e1c278800 nid=0x472c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "ForkJoinPool-1-worker-7" #114939 daemon prio=5 os_prio=0 tid=0x00007f8e44069000 nid=0x46c6 waiting on condition [0x00007f8db6ebe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dff16f20> (a scala.concurrent.forkjoin.ForkJoinPool) at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "Hashed wheel timer #2" #170 prio=5 os_prio=0 tid=0x00007f8df000f800 nid=0x9ab sleeping[0x00007f8db45a1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445) at org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New I/O boss #9" #171 prio=5 os_prio=0 tid=0x00007f8dec426800 nid=0x9aa runnable [0x00007f8db48a2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0abf1e0> (a sun.nio.ch.Util$3) - locked <0x00000005e0abf1d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0abef58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0abea58> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #8" #169 daemon prio=5 os_prio=0 tid=0x00007f8dec414000 nid=0x9a9 runnable [0x00007f8db49a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0ab6378> (a sun.nio.ch.Util$3) - locked <0x00000005e0ab6368> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0ab6140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0ab3b30> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #7" #168 daemon prio=5 os_prio=0 tid=0x00007f8dec337800 nid=0x9a8 runnable [0x00007f8db4aa4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0ab3830> (a sun.nio.ch.Util$3) - locked <0x00000005e0ab3820> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0ab35f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aafee0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #6" #167 daemon prio=5 os_prio=0 tid=0x00007f8dec336000 nid=0x9a7 runnable [0x00007f8db4ba5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0ababc0> (a sun.nio.ch.Util$3) - locked <0x00000005e0ababb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0aba988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aba3c0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #5" #166 daemon prio=5 os_prio=0 tid=0x00007f8dec334800 nid=0x9a6 runnable [0x00007f8db4ca6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0aafbe0> (a sun.nio.ch.Util$3) - locked <0x00000005e0aafbd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0aaf9a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aaf3e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #4" #165 daemon prio=5 os_prio=0 tid=0x00007f8dec332800 nid=0x9a5 runnable [0x00007f8db4da7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0ab8070> (a sun.nio.ch.Util$3) - locked <0x00000005e0ab8060> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0ab7e38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0ab7688> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #3" #164 daemon prio=5 os_prio=0 tid=0x00007f8dec31f000 nid=0x9a4 runnable [0x00007f8db4ea8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0aca8f0> (a sun.nio.ch.Util$3) - locked <0x00000005e0aca8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0aca6b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0ac9258> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #2" #163 daemon prio=5 os_prio=0 tid=0x00007f8dec31b800 nid=0x9a3 runnable [0x00007f8db4fa9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0ad59f0> (a sun.nio.ch.Util$3) - locked <0x00000005e0ad59e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0abe620> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0abaec0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "New I/O worker #1" #162 daemon prio=5 os_prio=0 tid=0x00007f8dec325000 nid=0x9a2 runnable [0x00007f8db50aa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0aae0d0> (a sun.nio.ch.Util$3) - locked <0x00000005e0aae0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0aade48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aac288> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Hashed wheel timer #1" #161 prio=5 os_prio=0 tid=0x00007f8dec302000 nid=0x9a1 sleeping[0x00007f8db51ab000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445) at org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "java-sdk-http-connection-reaper" #160 daemon prio=5 os_prio=0 tid=0x00007f8dec23c000 nid=0x9a0 waiting on condition [0x00007f8db54ac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188) Locked ownable synchronizers: - None "ConsumerFetcherThread-instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11_ip-10-2-2-20-1563138086768-885cab87-0-3" #159 prio=5 os_prio=0 tid=0x00007f8e18010800 nid=0x99f runnable [0x00007f8db5dad000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e0d621f0> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e0d621c0> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e0d62318> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e0d62950> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e0d628a8> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "ConsumerFetcherThread-instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11_ip-10-2-2-20-1563138086768-885cab87-0-1" #158 prio=5 os_prio=0 tid=0x00007f8e1800f000 nid=0x99e runnable [0x00007f8db5eae000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e0c3af50> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e0c3af20> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e0c3b0d8> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e0c3b718> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e0c3b670> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "ConsumerFetcherThread-instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11_ip-10-2-2-20-1563138086768-885cab87-0-2" #157 prio=5 os_prio=0 tid=0x00007f8e1800d000 nid=0x99d runnable [0x00007f8db5faf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e0b0eaa0> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e0b0ea70> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e0b0ebc8> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e0b76340> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e0b76298> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #156 daemon prio=5 os_prio=0 tid=0x00007f8dec04e800 nid=0x99c waiting on condition [0x00007f8db60af000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa6e98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aa6b68> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #155 daemon prio=5 os_prio=0 tid=0x00007f8dec04d000 nid=0x99b waiting on condition [0x00007f8db61b1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0a91fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0a91c78> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #154 daemon prio=5 os_prio=0 tid=0x00007f8dec04b000 nid=0x99a waiting on condition [0x00007f8db62b2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa6ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aa67b0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #153 daemon prio=5 os_prio=0 tid=0x00007f8dec049800 nid=0x999 waiting on condition [0x00007f8db63b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa6370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0a92030> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #152 daemon prio=5 os_prio=0 tid=0x00007f8dec047800 nid=0x998 waiting on condition [0x00007f8db64b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0a91bf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0a918c0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #151 daemon prio=5 os_prio=0 tid=0x00007f8dec046000 nid=0x997 waiting on condition [0x00007f8db65b5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa7608> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aa72d8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #150 daemon prio=5 os_prio=0 tid=0x00007f8dec044000 nid=0x996 waiting on condition [0x00007f8db66b6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa6728> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aa63f8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer institutionChange_1.0.0:instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11-0" #149 daemon prio=5 os_prio=0 tid=0x00007f8dec042800 nid=0x995 waiting on condition [0x00007f8db67b7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0aa7250> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005e0aa6f20> (a java.util.concurrent.ThreadPoolExecutor$Worker) "instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11_ip-10-2-2-20-1563138086768-885cab87-leader-finder-thread" #148 prio=5 os_prio=0 tid=0x00007f8dec040800 nid=0x994 waiting on condition [0x00007f8db68b8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0e1daa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at kafka.consumer.ConsumerFetcherManager$LeaderFinderThread.doWork(ConsumerFetcherManager.scala:61) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "instHostCache-ip-10-2-2-20c59a7856-0122-4587-8ebe-e9cd673f7b11_ip-10-2-2-20-1563138086768-885cab87_watcher_executor" #147 daemon prio=5 os_prio=0 tid=0x00007f8dec03a000 nid=0x993 runnable [0x00007f8db69b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0e17c10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.consumer.ZookeeperConsumerConnector$ZKRebalancerListener$$anon$1.run(ZookeeperConsumerConnector.scala:333) Locked ownable synchronizers: - None "ForkJoinPool-1-worker-5-EventThread" #146 daemon prio=5 os_prio=0 tid=0x00007f8dec021800 nid=0x992 waiting on condition [0x00007f8db6aba000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0e17140> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "ForkJoinPool-1-worker-5-SendThread(10.2.1.207:2181)" #145 daemon prio=5 os_prio=0 tid=0x00007f8dec020000 nid=0x991 runnable [0x00007f8db6bbb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e0e0fa60> (a sun.nio.ch.Util$3) - locked <0x00000005e0e0fa50> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e0e0f5e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ZkClient-EventThread-144-10.2.1.107:2181,10.2.1.207:2181,10.2.2.86:2181" #144 daemon prio=5 os_prio=0 tid=0x00007f8dec01e000 nid=0x990 waiting on condition [0x00007f8db6cbc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0e1d648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) Locked ownable synchronizers: - None "New Relic Segment Expiration Handler 2" #143 daemon prio=5 os_prio=0 tid=0x00007f8dc41e1800 nid=0x98f waiting on condition [0x00007f8db6dbd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c058d2b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Segment Expiration Handler" #138 daemon prio=5 os_prio=0 tid=0x00007f8e44074800 nid=0x98a waiting on condition [0x00007f8db72c2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c058d2b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-14" #137 prio=5 os_prio=0 tid=0x00007f8dd015c800 nid=0x989 waiting on condition [0x00007f8db73c3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-13" #136 prio=5 os_prio=0 tid=0x00000000020fc000 nid=0x988 waiting on condition [0x00007f8db74c4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-12" #135 prio=5 os_prio=0 tid=0x00007f8dcc204000 nid=0x987 waiting on condition [0x00007f8db75c5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-11" #134 prio=5 os_prio=0 tid=0x00007f8e4c0fe800 nid=0x986 waiting on condition [0x00007f8db76c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-10" #133 prio=5 os_prio=0 tid=0x00007f8e3c03b000 nid=0x985 waiting on condition [0x00007f8db77c7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-9" #132 prio=5 os_prio=0 tid=0x00007f8e3c039800 nid=0x984 waiting on condition [0x00007f8db78c8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-8" #131 prio=5 os_prio=0 tid=0x00000000020fb000 nid=0x983 waiting on condition [0x00007f8e49753000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-7" #130 prio=5 os_prio=0 tid=0x00000000020fa800 nid=0x982 waiting on condition [0x00007f8e49955000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "New Relic Token Expiration Handler" #129 daemon prio=5 os_prio=0 tid=0x00007f8e44183000 nid=0x967 waiting on condition [0x00007f8db7bc9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c058d048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-6" #128 prio=5 os_prio=0 tid=0x00007f8e4404c000 nid=0x966 waiting on condition [0x00007f8db7eca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "New Relic Harvest Service" #127 daemon prio=5 os_prio=0 tid=0x00007f8e100fa800 nid=0x965 waiting on condition [0x00007f8dc94fb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0c17178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.io.pinned-dispatcher-5" #126 prio=5 os_prio=0 tid=0x00007f8e4404b000 nid=0x964 runnable [0x00007f8db7fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfc20bb8> (a sun.nio.ch.Util$3) - locked <0x00000005dfc20ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005df5e6e98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at akka.io.SelectionHandler$ChannelRegistryImpl$$anon$3.tryRun(SelectionHandler.scala:146) at akka.io.SelectionHandler$ChannelRegistryImpl$Task.run(SelectionHandler.scala:286) at akka.io.SelectionHandler$ChannelRegistryImpl$$anon$3.run(SelectionHandler.scala:179) at akka.util.SerializedSuspendableExecutionContext.run$1(SerializedSuspendableExecutionContext.scala:72) at akka.util.SerializedSuspendableExecutionContext.run(SerializedSuspendableExecutionContext.scala:77) at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005df5e6868> (a java.util.concurrent.ThreadPoolExecutor$Worker) "bundle-183-ActorSystem-akka.actor.default-dispatcher-4" #125 prio=5 os_prio=0 tid=0x00007f8e3c021000 nid=0x963 waiting on condition [0x00007f8db88cc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-3" #124 prio=5 os_prio=0 tid=0x00007f8e1c247000 nid=0x962 waiting on condition [0x00007f8db89cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-akka.actor.default-dispatcher-2" #123 prio=5 os_prio=0 tid=0x00007f8e1c23b000 nid=0x961 waiting on condition [0x00007f8db8ace000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfc49ac8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Locked ownable synchronizers: - None "bundle-183-ActorSystem-scheduler-1" #122 prio=5 os_prio=0 tid=0x00007f8e1c1b7000 nid=0x960 sleeping[0x00007f8db8dcf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at akka.actor.LightArrayRevolverScheduler.waitNanos(LightArrayRevolverScheduler.scala:87) at akka.actor.LightArrayRevolverScheduler$$anon$4.nextTick(LightArrayRevolverScheduler.scala:271) at akka.actor.LightArrayRevolverScheduler$$anon$4.run(LightArrayRevolverScheduler.scala:241) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972_ip-10-2-2-20-1563138043857-a1079a37-0-3" #121 prio=5 os_prio=0 tid=0x00007f8e00486000 nid=0x95f runnable [0x00007f8db90d0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e0029fa0> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e0029f70> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e002a0c8> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e002a4d0> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e002a428> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972_ip-10-2-2-20-1563138043857-a1079a37-0-1" #120 prio=5 os_prio=0 tid=0x00007f8e0048a000 nid=0x95e runnable [0x00007f8db91d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e00274f0> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e00274c0> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e0027618> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e0028bc8> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e0027b00> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "ConsumerFetcherThread-uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972_ip-10-2-2-20-1563138043857-a1079a37-0-2" #119 prio=5 os_prio=0 tid=0x00007f8e006ae000 nid=0x95d runnable [0x00007f8db92d2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.Net.poll(Native Method) at sun.nio.ch.SocketChannelImpl.poll(SocketChannelImpl.java:954) - locked <0x00000005e0007f38> (a java.lang.Object) at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:204) - locked <0x00000005e0007f08> (a java.lang.Object) at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) - locked <0x00000005e00080c0> (a sun.nio.ch.SocketAdaptor$SocketInputStream) at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) - locked <0x00000005e0008518> (a java.lang.Object) at kafka.utils.Utils$.read(Utils.scala:375) at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) at kafka.network.Receive$class.readCompletely(Transmission.scala:56) at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:73) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) - locked <0x00000005e0008470> (a java.lang.Object) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:109) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:108) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:107) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:96) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:88) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #118 prio=5 os_prio=0 tid=0x00007f8e1c117000 nid=0x95c waiting on condition [0x00007f8db95d3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd513f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd510f0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #117 prio=5 os_prio=0 tid=0x00007f8e1c115000 nid=0x95b waiting on condition [0x00007f8db96d4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd4be70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd4bb68> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #116 prio=5 os_prio=0 tid=0x00007f8e1c113000 nid=0x95a waiting on condition [0x00007f8db97d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd50cd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd509d0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #115 prio=5 os_prio=0 tid=0x00007f8e1c111800 nid=0x959 waiting on condition [0x00007f8db98d6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd51068> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd50d60> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #114 prio=5 os_prio=0 tid=0x00007f8e1c10f800 nid=0x958 waiting on condition [0x00007f8db99d7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd4c200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd4bef8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #113 prio=5 os_prio=0 tid=0x00007f8e1c10d800 nid=0x957 waiting on condition [0x00007f8db9ad8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd50948> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd50640> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #112 prio=5 os_prio=0 tid=0x00007f8e1c10b800 nid=0x956 waiting on condition [0x00007f8db9bd9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd4c590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd4c288> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Consumer urimappings_1.0.0:uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972-0" #111 prio=5 os_prio=0 tid=0x00007f8e1c10a800 nid=0x955 waiting on condition [0x00007f8db9cda000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dfd51a90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:63) at kafka.consumer.ConsumerIterator.makeNext(ConsumerIterator.scala:33) at kafka.utils.IteratorTemplate.maybeComputeNext(IteratorTemplate.scala:66) at kafka.utils.IteratorTemplate.hasNext(IteratorTemplate.scala:58) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at kafka.utils.IteratorTemplate.foreach(IteratorTemplate.scala:32) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.consumer.KafkaStream.foreach(KafkaStream.scala:25) at com.tle.core.messaging.impl.MessageServiceImpl$$anonfun$com$tle$core$messaging$impl$MessageServiceImpl$$consumeStreamsInPool$1$$anon$3.run(MessageServiceImpl.scala:271) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005dfd51480> (a java.util.concurrent.ThreadPoolExecutor$Worker) "uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972_ip-10-2-2-20-1563138043857-a1079a37-leader-finder-thread" #110 prio=5 os_prio=0 tid=0x00007f8e1c107000 nid=0x954 waiting on condition [0x00007f8db9ddb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0031d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at kafka.consumer.ConsumerFetcherManager$LeaderFinderThread.doWork(ConsumerFetcherManager.scala:61) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:51) Locked ownable synchronizers: - None "uriCacheInvalidator-ip-10-2-2-20989d3047-fcf5-4235-aba9-829e526cd972_ip-10-2-2-20-1563138043857-a1079a37_watcher_executor" #109 prio=5 os_prio=0 tid=0x00007f8e1c0f2000 nid=0x953 runnable [0x00007f8db9edc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e002fc88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.consumer.ZookeeperConsumerConnector$ZKRebalancerListener$$anon$1.run(ZookeeperConsumerConnector.scala:333) Locked ownable synchronizers: - None "pool-10-thread-1-EventThread" #108 daemon prio=5 os_prio=0 tid=0x00007f8e1c0c5800 nid=0x952 waiting on condition [0x00007f8db9fdd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e002f1a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-10-thread-1-SendThread(10.2.1.107:2181)" #107 daemon prio=5 os_prio=0 tid=0x00007f8e1c0c3800 nid=0x951 runnable [0x00007f8dba0de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005e002daf0> (a sun.nio.ch.Util$3) - locked <0x00000005e002dae0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005e002d678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ZkClient-EventThread-106-10.2.1.107:2181,10.2.1.207:2181,10.2.2.86:2181" #106 daemon prio=5 os_prio=0 tid=0x00007f8e1c0c7800 nid=0x950 waiting on condition [0x00007f8dba1df000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005e0031900> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) Locked ownable synchronizers: - None "I/O dispatcher 4" #105 prio=5 os_prio=0 tid=0x00007f8dd41a1000 nid=0x94f runnable [0x00007f8dbace0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfd52318> (a sun.nio.ch.Util$3) - locked <0x00000005dfd52308> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd521d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 3" #104 prio=5 os_prio=0 tid=0x00007f8dd416a000 nid=0x94e runnable [0x00007f8dbade1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfd5a890> (a sun.nio.ch.Util$3) - locked <0x00000005dfd5a880> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd5a748> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 2" #103 prio=5 os_prio=0 tid=0x00007f8dd4168000 nid=0x94d runnable [0x00007f8dbaee2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfd5bd78> (a sun.nio.ch.Util$3) - locked <0x00000005dfd5bd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd5bc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "I/O dispatcher 1" #102 prio=5 os_prio=0 tid=0x00007f8dd4167000 nid=0x94c runnable [0x00007f8dbafe3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfd5c250> (a sun.nio.ch.Util$3) - locked <0x00000005dfd5c240> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd5c108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255) at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-18-thread-1" #101 prio=5 os_prio=0 tid=0x00007f8e2857c800 nid=0x94b runnable [0x00007f8dbb0e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dfd51f38> (a sun.nio.ch.Util$3) - locked <0x00000005dfd51f28> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dfd51df0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:340) at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221) at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-6" #95 prio=5 os_prio=0 tid=0x00007f8e282b9800 nid=0x94a runnable [0x00007f8dc03ea000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005daf6a300> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005daf8c2b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005daf6a268> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-5" #94 prio=5 os_prio=0 tid=0x00007f8e2829d800 nid=0x949 runnable [0x00007f8dc04eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005daf8db60> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005daf8fbe0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005daf8dac8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-4" #93 prio=5 os_prio=0 tid=0x00007f8e2829c000 nid=0x948 runnable [0x00007f8dc05ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005daf91488> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005daf93508> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005daf913f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-3" #92 prio=5 os_prio=0 tid=0x00007f8e28298000 nid=0x947 runnable [0x00007f8dc06ed000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005daf94db0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005daf96e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005daf94d18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-2" #91 prio=5 os_prio=0 tid=0x00007f8e28297000 nid=0x946 runnable [0x00007f8dc07ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005daf986d8> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005dafa2748> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005daf98640> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-1" #90 prio=5 os_prio=0 tid=0x00007f8e28295800 nid=0x945 runnable [0x00007f8dc18f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dafa3ff0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005dafa6070> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dafa3f58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "threadDeathWatcher-2-1" #100 daemon prio=1 os_prio=0 tid=0x00007f8e4c04b000 nid=0x944 sleeping[0x00007f8dc29f2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:137) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-timeouter-0" #97 prio=5 os_prio=0 tid=0x00007f8e4c03a800 nid=0x943 sleeping[0x00007f8dc2af3000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:461) at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:360) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-nio-worker-0" #89 prio=5 os_prio=0 tid=0x00007f8e281d3000 nid=0x942 runnable [0x00007f8dc2df4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005dafa7918> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000005dafa9998> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005dafa7880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-scheduled-task-worker-0" #98 prio=5 os_prio=0 tid=0x00007f8e28193000 nid=0x940 waiting on condition [0x00007f8dc2ef5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005dafab1b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cluster1-connection-reaper-0" #88 prio=5 os_prio=0 tid=0x00007f8e280d8000 nid=0x93f waiting on condition [0x00007f8dc2ff6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005da08e7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-4" #87 prio=5 os_prio=0 tid=0x00007f8e1c001800 nid=0x93e waiting on condition [0x00007f8dc38f7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d89c99d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "CuratorFramework-0" #85 daemon prio=5 os_prio=0 tid=0x00007f8de0ff0000 nid=0x93c waiting on condition [0x00007f8dc3af9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d84feb88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.DelayQueue.take(DelayQueue.java:211) at java.util.concurrent.DelayQueue.take(DelayQueue.java:70) at com.netflix.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:632) at com.netflix.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:53) at com.netflix.curator.framework.imps.CuratorFrameworkImpl$3.call(CuratorFrameworkImpl.java:241) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d84fed88> (a java.util.concurrent.ThreadPoolExecutor$Worker) "pool-7-thread-1-EventThread" #84 daemon prio=5 os_prio=0 tid=0x00007f8de0fee000 nid=0x93b waiting on condition [0x00007f8dc3bfa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d84fe428> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-7-thread-1-SendThread(10.2.1.207:2181)" #83 daemon prio=5 os_prio=0 tid=0x00007f8de0fed000 nid=0x93a runnable [0x00007f8dc3cfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d84fca08> (a sun.nio.ch.Util$3) - locked <0x00000005d84fc9f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d84fc580> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ConnectionStateManager-0" #82 daemon prio=5 os_prio=0 tid=0x00007f8de0fda800 nid=0x939 waiting on condition [0x00007f8dc3dfc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d84ff2e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at com.netflix.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:170) at com.netflix.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:40) at com.netflix.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:104) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d84ff4e8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ProducerSendThread-" #81 prio=5 os_prio=0 tid=0x00007f8de0fc6800 nid=0x938 runnable [0x00007f8dc3efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d85224e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at scala.collection.immutable.Stream$.continually(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007251bb950> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007251bb980> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream.foreach(Stream.scala:595) at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:66) at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:44) Locked ownable synchronizers: - None "ProducerSendThread-" #80 prio=5 os_prio=0 tid=0x00007f8de0fba800 nid=0x937 runnable [0x00007f8dc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8521448> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$1.apply(ProducerSendThread.scala:65) at scala.collection.immutable.Stream$.continually(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$$anonfun$continually$1.apply(Stream.scala:1279) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007251c9c28> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$$anonfun$takeWhile$1.apply(Stream.scala:941) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1233) - locked <0x00000007251c9c58> (a scala.collection.immutable.Stream$Cons) at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1223) at scala.collection.immutable.Stream.foreach(Stream.scala:595) at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:66) at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:44) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #79 daemon prio=5 os_prio=0 tid=0x00007f8de0fb6000 nid=0x936 waiting on condition [0x00007f8dc81ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8520b98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #78 daemon prio=5 os_prio=0 tid=0x00007f8de0faf800 nid=0x935 waiting on condition [0x00007f8dc82cb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d8520b98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-7-thread-1-EventThread" #76 daemon prio=5 os_prio=0 tid=0x00007f8de00fb000 nid=0x934 waiting on condition [0x00007f8dc85cc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d851ddc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "pool-7-thread-1-SendThread(10.2.1.107:2181)" #75 daemon prio=5 os_prio=0 tid=0x00007f8de00f9800 nid=0x933 runnable [0x00007f8dc86cd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d85027d0> (a sun.nio.ch.Util$3) - locked <0x00000005d85027c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d8502358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ZkClient-EventThread-74-10.2.1.107:2181,10.2.1.207:2181,10.2.2.86:2181" #74 daemon prio=5 os_prio=0 tid=0x00007f8de01bb800 nid=0x932 waiting on condition [0x00007f8dc87ce000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d851e690> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) Locked ownable synchronizers: - None "LocatorPool-3-EventThread" #73 daemon prio=5 os_prio=0 tid=0x00007f8de4057000 nid=0x931 waiting on condition [0x00007f8dc8acf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d937aca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "LocatorPool-3-SendThread(10.2.1.207:2181)" #72 daemon prio=5 os_prio=0 tid=0x00007f8de4056000 nid=0x930 runnable [0x00007f8dc8bd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000005d93794a8> (a sun.nio.ch.Util$3) - locked <0x00000005d9379498> (a java.util.Collections$UnmodifiableSet) - locked <0x00000005d9379030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "LocatorPool-3" #71 prio=5 os_prio=0 tid=0x00007f8de0163000 nid=0x92f waiting on condition [0x00007f8dc8cd1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d89c99d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-2" #69 prio=5 os_prio=0 tid=0x00007f8de00f1000 nid=0x92d waiting on condition [0x00007f8dc8ff8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d89c99d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LocatorPool-1" #68 prio=5 os_prio=0 tid=0x00007f8de00ef800 nid=0x92c waiting on condition [0x00007f8dc90f9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d89c99d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixStartLevel" #65 daemon prio=5 os_prio=0 tid=0x00007f8dd01a3800 nid=0x929 in Object.wait() [0x00007f8dca518000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkStartLevelImpl.run(FrameworkStartLevelImpl.java:283) - locked <0x00000005dafaebe8> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixFrameworkWiring" #64 daemon prio=5 os_prio=0 tid=0x00007f8dd0194000 nid=0x928 in Object.wait() [0x00007f8dca619000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005db0ee678> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkWiringImpl.run(FrameworkWiringImpl.java:172) - locked <0x00000005db0ee678> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-5-thread-9" #63 prio=5 os_prio=0 tid=0x00007f8dd80fa000 nid=0x927 in Object.wait() [0x00007f8dcb883000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.felix.framework.util.ThreadGate.await(ThreadGate.java:79) - locked <0x00000005db0ee808> (a org.apache.felix.framework.util.ThreadGate) at org.apache.felix.framework.Felix.waitForStop(Felix.java:1068) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26$$anonfun$apply$30.apply(OsgiTasks.scala:284) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26$$anonfun$apply$30.apply(OsgiTasks.scala:284) at osgifelix.FelixRunner$.embed(FelixRunner.scala:60) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26.apply(OsgiTasks.scala:282) at osgifelix.OsgiTasks$$anonfun$osgiRunTask$1$$anonfun$apply$25$$anonfun$apply$26.apply(OsgiTasks.scala:273) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:47) at sbt.$tilde$greater$$anonfun$$u2219$1.apply(TypeFunctions.scala:40) at sbt.std.Transform$$anon$4.work(System.scala:63) at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:228) at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:228) at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:17) at sbt.Execute.work(Execute.scala:237) at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:228) at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:228) at sbt.ConcurrentRestrictions$$anon$4$$anonfun$1.apply(ConcurrentRestrictions.scala:159) at sbt.CompletionService$$anon$2.call(CompletionService.scala:28) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005d349f448> (a java.util.concurrent.ThreadPoolExecutor$Worker) "FelixStartLevel" #61 daemon prio=5 os_prio=0 tid=0x00007f8de0100800 nid=0x925 in Object.wait() [0x00007f8e481ce000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005d6a7f838> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkStartLevelImpl.run(FrameworkStartLevelImpl.java:283) - locked <0x00000005d6a7f838> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixFrameworkWiring" #60 daemon prio=5 os_prio=0 tid=0x00007f8de01cc800 nid=0x924 in Object.wait() [0x00007f8e482d3000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000005d6a80078> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.FrameworkWiringImpl.run(FrameworkWiringImpl.java:172) - locked <0x00000005d6a80078> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "FelixDispatchQueue" #59 prio=5 os_prio=0 tid=0x00007f8de0109000 nid=0x923 in Object.wait() [0x00007f8e483d4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.felix.framework.util.EventDispatcher.run(EventDispatcher.java:1118) - locked <0x00000005d6a80380> (a java.util.ArrayList) at org.apache.felix.framework.util.EventDispatcher.access$000(EventDispatcher.java:55) at org.apache.felix.framework.util.EventDispatcher$1.run(EventDispatcher.java:102) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #33 daemon prio=9 os_prio=0 tid=0x00007f8e6c8f2000 nid=0x7d6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread2" #32 daemon prio=9 os_prio=0 tid=0x00007f8e6c8ef800 nid=0x7d5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #31 daemon prio=9 os_prio=0 tid=0x00007f8e6c8ed800 nid=0x7d4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #30 daemon prio=9 os_prio=0 tid=0x00007f8e6c8ed000 nid=0x7d3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "New Relic RPM Connection Service" #29 daemon prio=5 os_prio=0 tid=0x00007f8e6c8ea800 nid=0x7d2 waiting on condition [0x00007f8e50c77000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0132720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Deadlock Detector" #28 daemon prio=5 os_prio=0 tid=0x00007f8e6c8e7800 nid=0x7d1 waiting on condition [0x00007f8e50d78000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0c1b0d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Sampler Service" #27 daemon prio=5 os_prio=0 tid=0x00007f8e6c8e0000 nid=0x7c1 waiting on condition [0x00007f8e50974000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0b30440> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Transaction Service Processor" #26 daemon prio=5 os_prio=0 tid=0x00007f8e6c806800 nid=0x650 waiting on condition [0x00007f8e50e79000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0000730> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Utilization Service 2" #25 daemon prio=5 os_prio=0 tid=0x00007f8e6c801800 nid=0x64f waiting on condition [0x00007f8e50b76000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c000bd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Utilization Service" #24 daemon prio=5 os_prio=0 tid=0x00007f8e6c800800 nid=0x64e waiting on condition [0x00007f8e50a75000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c000bd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "New Relic Retransformer" #23 daemon prio=5 os_prio=0 tid=0x00007f8e6c7e4000 nid=0x64d waiting on condition [0x00007f8e5107b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005c0011c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f8e6c17e800 nid=0x562 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f8e6c14c000 nid=0x561 in Object.wait() [0x00007f8e52514000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000005c000c160> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f8e6c147800 nid=0x560 in Object.wait() [0x00007f8e52615000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000005c000c318> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f8e6c00a800 nid=0x556 waiting on condition [0x00007f8e725e5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000005d6a7f488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:193) at sbt.ConcurrentRestrictions$$anon$4.take(ConcurrentRestrictions.scala:188) at sbt.Execute.next$1(Execute.scala:85) at sbt.Execute.processAll(Execute.scala:88) at sbt.Execute.runKeep(Execute.scala:68) at sbt.EvaluateTask$.liftedTree1$1(EvaluateTask.scala:359) at sbt.EvaluateTask$.run$1(EvaluateTask.scala:358) at sbt.EvaluateTask$.runTask(EvaluateTask.scala:378) at sbt.Aggregation$$anonfun$3.apply(Aggregation.scala:64) at sbt.Aggregation$$anonfun$3.apply(Aggregation.scala:62) at sbt.EvaluateTask$.withStreams(EvaluateTask.scala:314) at sbt.Aggregation$.timedRun(Aggregation.scala:62) at sbt.Aggregation$.runTasks(Aggregation.scala:71) at sbt.Aggregation$$anonfun$applyDynamicTasks$1.apply(Aggregation.scala:112) at sbt.Aggregation$$anonfun$applyDynamicTasks$1.apply(Aggregation.scala:110) at sbt.Command$$anonfun$applyEffect$2$$anonfun$apply$3.apply(Command.scala:61) at sbt.Command$$anonfun$applyEffect$2$$anonfun$apply$3.apply(Command.scala:61) at sbt.Act$$anonfun$sbt$Act$$actParser0$1$$anonfun$sbt$Act$$anonfun$$evaluate$1$1$$anonfun$apply$10.apply(Act.scala:244) at sbt.Act$$anonfun$sbt$Act$$actParser0$1$$anonfun$sbt$Act$$anonfun$$evaluate$1$1$$anonfun$apply$10.apply(Act.scala:241) at sbt.Command$.process(Command.scala:93) at sbt.MainLoop$$anonfun$1$$anonfun$apply$1.apply(MainLoop.scala:96) at sbt.MainLoop$$anonfun$1$$anonfun$apply$1.apply(MainLoop.scala:96) at sbt.State$$anon$1.process(State.scala:184) at sbt.MainLoop$$anonfun$1.apply(MainLoop.scala:96) at sbt.MainLoop$$anonfun$1.apply(MainLoop.scala:96) at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:17) at sbt.MainLoop$.next(MainLoop.scala:96) at sbt.MainLoop$.run(MainLoop.scala:89) at sbt.MainLoop$$anonfun$runWithNewLog$1.apply(MainLoop.scala:68) at sbt.MainLoop$$anonfun$runWithNewLog$1.apply(MainLoop.scala:63) at sbt.Using.apply(Using.scala:24) at sbt.MainLoop$.runWithNewLog(MainLoop.scala:63) at sbt.MainLoop$.runAndClearLast(MainLoop.scala:46) at sbt.MainLoop$.runLoggedLoop(MainLoop.scala:30) at sbt.MainLoop$.runLogged(MainLoop.scala:22) at sbt.StandardMain$.runManaged(Main.scala:54) at sbt.xMain.run(Main.scala:29) at xsbt.boot.Launch$$anonfun$run$1.apply(Launch.scala:109) at xsbt.boot.Launch$.withContextLoader(Launch.scala:128) at xsbt.boot.Launch$.run(Launch.scala:109) at xsbt.boot.Launch$$anonfun$apply$1.apply(Launch.scala:35) at xsbt.boot.Launch$.launch(Launch.scala:117) at xsbt.boot.Launch$.apply(Launch.scala:18) at xsbt.boot.Boot$.runImpl(Boot.scala:41) at xsbt.boot.Boot$.main(Boot.scala:17) at xsbt.boot.Boot.main(Boot.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f8e6c13f800 nid=0x55f runnable "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f8e6c020000 nid=0x557 runnable "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f8e6c021800 nid=0x558 runnable "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f8e6c023800 nid=0x559 runnable "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f8e6c025000 nid=0x55a runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f8e6c8d3000 nid=0x7d7 waiting on condition JNI global references: 328