Process Thread Dump: 26 active threads Thread 479 (hiveserver2-web-479): State: RUNNABLE Blocked count: 10 Wtaited count: 487 Stack: sun.management.ThreadImpl.getThreadInfo1(Native Method) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:185) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:144) org.apache.hive.http.StackServlet.printThreadInfo(StackServlet.java:67) org.apache.hive.http.StackServlet.doGet(StackServlet.java:49) javax.servlet.http.HttpServlet.service(HttpServlet.java:735) javax.servlet.http.HttpServlet.service(HttpServlet.java:848) org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565) org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479) org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:119) org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:521) org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:227) org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031) org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406) org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186) org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965) org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117) org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111) org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:312) org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:250) Thread 340 (HiveServer2-Handler-Pool: Thread-340): State: RUNNABLE Blocked count: 9 Wtaited count: 17 Stack: java.net.SocketInputStream.socketRead0(Native Method) java.net.SocketInputStream.socketRead(SocketInputStream.java:116) java.net.SocketInputStream.read(SocketInputStream.java:171) java.net.SocketInputStream.read(SocketInputStream.java:141) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:127) org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:376) org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:453) org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:435) org.apache.thrift.transport.TSaslServerTransport.read(TSaslServerTransport.java:41) org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:27) org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) Thread 53 (HiveServer2-Handler-Pool: Thread-53): State: WAITING Blocked count: 10 Wtaited count: 20 Waiting on java.util.concurrent.SynchronousQueue$TransferStack@45e92e85 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Thread 44 (hiveserver2-web-44 Acceptor5 SelectChannelConnector@0.0.0.0:10002): State: BLOCKED Blocked count: 42 Wtaited count: 0 Blocked on java.lang.Object@67ac8212 Blocked by 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002) Stack: sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:232) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 43 (DestroyJavaVM): State: RUNNABLE Blocked count: 0 Wtaited count: 0 Stack: Thread 42 (org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@119b0892): State: TIMED_WAITING Blocked count: 0 Wtaited count: 50675 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:181) java.lang.Thread.run(Thread.java:750) Thread 41 (hiveserver2-web-41 Acceptor4 SelectChannelConnector@0.0.0.0:10002): State: BLOCKED Blocked count: 42 Wtaited count: 0 Blocked on java.lang.Object@67ac8212 Blocked by 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002) Stack: sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:232) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002): State: RUNNABLE Blocked count: 42 Wtaited count: 0 Stack: sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 39 (hiveserver2-web-39 Acceptor2 SelectChannelConnector@0.0.0.0:10002): State: BLOCKED Blocked count: 22 Wtaited count: 0 Blocked on java.lang.Object@67ac8212 Blocked by 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002) Stack: sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:232) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 38 (hiveserver2-web-38 Acceptor1 SelectChannelConnector@0.0.0.0:10002): State: BLOCKED Blocked count: 22 Wtaited count: 1 Blocked on java.lang.Object@67ac8212 Blocked by 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002) Stack: sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:232) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 37 (hiveserver2-web-37 Acceptor0 SelectChannelConnector@0.0.0.0:10002): State: BLOCKED Blocked count: 42 Wtaited count: 1 Blocked on java.lang.Object@67ac8212 Blocked by 40 (hiveserver2-web-40 Acceptor3 SelectChannelConnector@0.0.0.0:10002) Stack: sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:232) org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:95) org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:933) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 36 (hiveserver2-web-36 Selector5): State: RUNNABLE Blocked count: 1 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 35 (hiveserver2-web-35 Selector4): State: RUNNABLE Blocked count: 1 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 34 (hiveserver2-web-34 Selector3): State: RUNNABLE Blocked count: 2 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 33 (hiveserver2-web-33 Selector2): State: RUNNABLE Blocked count: 1 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 32 (hiveserver2-web-32 Selector1): State: RUNNABLE Blocked count: 1 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 31 (hiveserver2-web-31 Selector0): State: RUNNABLE Blocked count: 2 Wtaited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:564) org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:285) org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:599) org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:534) java.lang.Thread.run(Thread.java:750) Thread 30 (HashSessionScavenger-0): State: TIMED_WAITING Blocked count: 0 Wtaited count: 845 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Thread 29 (Thread-7): State: RUNNABLE Blocked count: 2 Wtaited count: 0 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) java.net.ServerSocket.implAccept(ServerSocket.java:560) java.net.ServerSocket.accept(ServerSocket.java:528) org.apache.thrift.transport.TServerSocket.acceptImpl(TServerSocket.java:129) org.apache.thrift.transport.TServerSocket.acceptImpl(TServerSocket.java:35) org.apache.thrift.transport.TServerTransport.accept(TServerTransport.java:60) org.apache.thrift.server.TThreadPoolServer.serve(TThreadPoolServer.java:162) org.apache.hive.service.cli.thrift.ThriftBinaryCLIService.run(ThriftBinaryCLIService.java:164) java.lang.Thread.run(Thread.java:750) Thread 28 (HiveServer2-Background-Pool: Thread-28): State: TIMED_WAITING Blocked count: 0 Wtaited count: 2 Stack: java.lang.Object.wait(Native Method) org.apache.hive.service.cli.session.SessionManager$7.sleepFor(SessionManager.java:289) org.apache.hive.service.cli.session.SessionManager$7.run(SessionManager.java:282) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Thread 24 (client DomainSocketWatcher): State: RUNNABLE Blocked count: 0 Wtaited count: 0 Stack: org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:503) java.lang.Thread.run(Thread.java:750) Thread 23 (org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner): State: WAITING Blocked count: 2 Wtaited count: 3 Waiting on java.lang.ref.ReferenceQueue$Lock@5c19423f Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3712) java.lang.Thread.run(Thread.java:750) Thread 22 (Log4j2-AsyncLogger[AsyncContext@61443d8f]1): State: TIMED_WAITING Blocked count: 0 Wtaited count: 2511642 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) com.lmax.disruptor.TimeoutBlockingWaitStrategy.waitFor(TimeoutBlockingWaitStrategy.java:37) com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55) com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Thread 4 (Signal Dispatcher): State: RUNNABLE Blocked count: 0 Wtaited count: 0 Stack: Thread 3 (Finalizer): State: WAITING Blocked count: 46 Wtaited count: 31 Waiting on java.lang.ref.ReferenceQueue$Lock@767f675d Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188) Thread 2 (Reference Handler): State: WAITING Blocked count: 31 Wtaited count: 30 Waiting on java.lang.ref.Reference$Lock@607c4794 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) java.lang.ref.Reference.tryHandlePending(Reference.java:191) java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)