Chen Luo created ASTERIXDB-2432: ----------------------------------- Summary: HTTP server hangs with lot of threads waiting for completion Key: ASTERIXDB-2432 URL: https://issues.apache.org/jira/browse/ASTERIXDB-2432 Project: Apache AsterixDB Issue Type: Bug Components: API - HTTP API Reporter: Chen Luo
Cloudberry experienced an issue that the HTTP port 19002 (for HTTP API) is not responsive, while everything else works fine. It seems that a lot of CC threads are waiting for job completions, and the HTTP server hangs. The complete stack trace is attached: {code} 2018-07-30 10:56:14 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.0-b70 mixed mode): "Attach Listener" #5244 daemon prio=9 os_prio=0 tid=0x00007f0268349800 nid=0x4bf7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "HttpExecutor(port:19001)-15" #5193 prio=10 os_prio=0 tid=0x00007f023c00c000 nid=0x4054 waiting on condition [0x00007f01fb703000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-14" #5192 prio=10 os_prio=0 tid=0x00007f0220019000 nid=0x4053 waiting on condition [0x00007f01fb804000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-13" #5191 prio=10 os_prio=0 tid=0x00007f0234005000 nid=0x4052 waiting on condition [0x00007f01fb905000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-12" #5190 prio=10 os_prio=0 tid=0x00007f0220017000 nid=0x4051 waiting on condition [0x00007f01fba06000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "TCPEndpoint IO Thread" #5159 daemon prio=5 os_prio=0 tid=0x00007f020008b000 nid=0x394d runnable [0x00007f01fbb07000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006cc4811e0> (a sun.nio.ch.Util$2) - locked <0x00000006cc4811d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006cc4811f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:127) Locked ownable synchronizers: - None "IPC Network Listener Thread [/0:0:0:0:0:0:0:0:47009]" #5158 daemon prio=10 os_prio=0 tid=0x00007f02000bc000 nid=0x394c runnable [0x00007f01fbc08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006cc4e2ce0> (a sun.nio.ch.Util$2) - locked <0x00000006cc4e2cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006cc4e2cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:216) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-11" #5157 prio=10 os_prio=0 tid=0x00007f023400b000 nid=0x394b waiting on condition [0x00007f01fbd09000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-10" #5156 prio=10 os_prio=0 tid=0x00007f0234009800 nid=0x3940 waiting on condition [0x00007f01fbe0a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Java2D Disposer" #5155 daemon prio=10 os_prio=0 tid=0x00007f02000b0000 nid=0x393f in Object.wait() [0x00007f01fbf0b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142) - locked <0x00000006ccba6c78> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158) at sun.java2d.Disposer.run(Disposer.java:150) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-9" #5153 prio=10 os_prio=0 tid=0x00007f0234007800 nid=0x393e waiting on condition [0x00007f01fc4e5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-8" #5152 prio=10 os_prio=0 tid=0x00007f0220015800 nid=0x393d waiting on condition [0x00007f01fc5e6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-7" #5151 prio=10 os_prio=0 tid=0x00007f0258092000 nid=0x393c waiting on condition [0x00007f01fc6e7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-6" #5150 prio=10 os_prio=0 tid=0x00007f0254005000 nid=0x393b waiting on condition [0x00007f01fc7e8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-5" #5149 prio=10 os_prio=0 tid=0x00007f0234006000 nid=0x393a waiting on condition [0x00007f01fc8e9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-4" #5148 prio=10 os_prio=0 tid=0x00007f0220015000 nid=0x3939 waiting on condition [0x00007f01fc9ea000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-3" #5147 prio=10 os_prio=0 tid=0x00007f027801a800 nid=0x3938 waiting on condition [0x00007f01fcaeb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-2" #5146 prio=10 os_prio=0 tid=0x00007f027c037000 nid=0x3937 waiting on condition [0x00007f01fcbec000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-1" #5145 prio=10 os_prio=0 tid=0x00007f0254002000 nid=0x3936 waiting on condition [0x00007f01fcced000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HttpExecutor(port:19001)-0" #5144 prio=10 os_prio=0 tid=0x00007f02541cf800 nid=0x3935 waiting on condition [0x00007f01fcdee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca6ba290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Executor-96:ClusterController" #4578 daemon prio=10 os_prio=0 tid=0x00007f0244125800 nid=0x3ccd waiting on condition [0x00007f01fd2f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006c9b81bc0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Executor-94:ClusterController" #3831 daemon prio=10 os_prio=0 tid=0x00007f0210051800 nid=0x1865 in Object.wait() [0x00007f01fd5f6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbd8eb40> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbd8e8d0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-93:ClusterController" #3700 daemon prio=10 os_prio=0 tid=0x00007f0244122800 nid=0x7a3a in Object.wait() [0x00007f0286ef1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbd7a800> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbbe61e8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-92:ClusterController" #3459 daemon prio=10 os_prio=0 tid=0x00007f0244122000 nid=0x459b in Object.wait() [0x00007f01fd6f7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbbe6238> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbbcea08> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-91:ClusterController" #3218 daemon prio=10 os_prio=0 tid=0x00007f0244121800 nid=0x114a in Object.wait() [0x00007f01fd4f5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbbd7248> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbbbc978> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-90:ClusterController" #3007 daemon prio=10 os_prio=0 tid=0x00007f0244120800 nid=0x62ac in Object.wait() [0x00007f02042f6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbbb5100> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb662a8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-89:ClusterController" #3006 daemon prio=10 os_prio=0 tid=0x00007f0244120000 nid=0x62a8 in Object.wait() [0x00007f01fddfc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb72428> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb724a8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-88:ClusterController" #2975 daemon prio=10 os_prio=0 tid=0x00007f024411f000 nid=0x5afb in Object.wait() [0x00007f02043f7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb4e2f8> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb7bd98> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-87:ClusterController" #2734 daemon prio=10 os_prio=0 tid=0x00007f024411e800 nid=0x26dc in Object.wait() [0x00007f02872f5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb7bec0> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb35778> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-86:ClusterController" #2493 daemon prio=10 os_prio=0 tid=0x00007f024411d800 nid=0x7148 in Object.wait() [0x00007f01fdbfa000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb357a8> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb1d4d8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-85:ClusterController" #2252 daemon prio=10 os_prio=0 tid=0x00007f024411d000 nid=0x3c53 in Object.wait() [0x00007f02041f5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb1d508> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbb05688> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-84:ClusterController" #2011 daemon prio=10 os_prio=0 tid=0x00007f024411c000 nid=0x81a in Object.wait() [0x00007f01fd9f8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbb05760> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbaecd28> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-83:ClusterController" #1770 daemon prio=10 os_prio=0 tid=0x00007f0244109800 nid=0x51d2 in Object.wait() [0x00007f02871f4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbaece00> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cbad64a8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-82:ClusterController" #1589 daemon prio=10 os_prio=0 tid=0x00007f024411b800 nid=0x2b21 in Object.wait() [0x00007f02873f6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cbad6580> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cba9c268> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-81:ClusterController" #1578 daemon prio=10 os_prio=0 tid=0x00007f024411a800 nid=0x2876 in Object.wait() [0x00007f01fdaf9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cba9c298> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb9782c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-80:ClusterController" #1567 daemon prio=10 os_prio=0 tid=0x00007f0244119800 nid=0x268a in Object.wait() [0x00007f0286bee000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cb978470> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb9784f0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Executor-43:ClusterController" #750 daemon prio=10 os_prio=0 tid=0x00007f024411a000 nid=0x763f in Object.wait() [0x00007f01fdcfb000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:223) - locked <0x00000006cb978520> (a org.apache.hyracks.control.cc.job.JobRun) at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:53) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb9643e8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #68 prio=10 os_prio=0 tid=0x00007f0244113800 nid=0x6349 in Object.wait() [0x00007f02870f2000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbad4c40> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb8a4c10> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #67 prio=10 os_prio=0 tid=0x00007f0254499800 nid=0x6348 in Object.wait() [0x00007f02874f6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbaf7938> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb728668> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #66 prio=10 os_prio=0 tid=0x00007f023c007800 nid=0x6347 in Object.wait() [0x00007f0286ff1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cb987780> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.common.utils.JobUtils.runJob(JobUtils.java:46) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2487) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2482) at org.apache.asterix.app.translator.QueryTranslator.handleInsertUpsertStatement(QueryTranslator.java:1844) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:346) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb7401f8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #62 prio=10 os_prio=0 tid=0x00007f0258032800 nid=0x6314 in Object.wait() [0x00007f0286aec000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbb2ba00> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb6e1368> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #61 prio=10 os_prio=0 tid=0x00007f023c006800 nid=0x6313 in Object.wait() [0x00007f02869eb000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbbd5508> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb6ddf50> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #60 prio=10 os_prio=0 tid=0x00007f0254094000 nid=0x6312 in Object.wait() [0x00007f02868ea000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbb7a1c8> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb6f3260> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #59 prio=10 os_prio=0 tid=0x00007f0244116800 nid=0x6311 in Object.wait() [0x00007f025c9d4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbba8ac8> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb6efda0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #58 prio=10 os_prio=0 tid=0x00007f0234003800 nid=0x62d0 in Object.wait() [0x00007f0286def000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbaa3f60> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.common.utils.JobUtils.runJob(JobUtils.java:46) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2487) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2482) at org.apache.asterix.app.translator.QueryTranslator.handleInsertUpsertStatement(QueryTranslator.java:1844) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:346) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb60ac98> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #57 prio=10 os_prio=0 tid=0x00007f027c03b800 nid=0x628d in Object.wait() [0x00007f02048f7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbbe5f28> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb5bd670> (a java.util.concurrent.ThreadPoolExecutor$Worker) "QueryTranslator" #56 prio=10 os_prio=0 tid=0x00007f027c039800 nid=0x628c in Object.wait() [0x00007f02049f8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbbc37a8> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb641a90> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-8" #55 prio=10 os_prio=0 tid=0x00007f0224010800 nid=0x628b runnable [0x00007f0204afa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca523fb0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca524010> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca523f18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "nioEventLoopGroup-5-7" #54 prio=10 os_prio=0 tid=0x00007f022400f000 nid=0x628a runnable [0x00007f0204bfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca5248b0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca63b9b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca524818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "TCPEndpoint IO Thread" #53 daemon prio=5 os_prio=0 tid=0x00007f0268130800 nid=0x6270 runnable [0x00007f025c1cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006cb4c4d88> (a sun.nio.ch.Util$2) - locked <0x00000006cb4c4d78> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006cb4c4c60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:127) Locked ownable synchronizers: - None "IPC Network Listener Thread [/0:0:0:0:0:0:0:0:48558]" #52 daemon prio=10 os_prio=0 tid=0x00007f02680e0000 nid=0x626f runnable [0x00007f025c2d0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006cb4c4270> (a sun.nio.ch.Util$2) - locked <0x00000006cb4c4260> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006cb2c0bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:216) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196) Locked ownable synchronizers: - None "QueryTranslator" #51 prio=10 os_prio=0 tid=0x00007f0258103800 nid=0x626e in Object.wait() [0x00007f025c3d0000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cb980558> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.common.utils.JobUtils.runJob(JobUtils.java:46) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2487) at org.apache.asterix.app.translator.QueryTranslator.runJob(QueryTranslator.java:2482) at org.apache.asterix.app.translator.QueryTranslator.handleInsertUpsertStatement(QueryTranslator.java:1844) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:346) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cadf2d40> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-6" #50 prio=10 os_prio=0 tid=0x00007f022400d000 nid=0x626d runnable [0x00007f025c4d2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca63d270> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca63d2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca63d1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "QueryTranslator" #49 prio=10 os_prio=0 tid=0x00007f02541ac800 nid=0x6269 in Object.wait() [0x00007f025c5d2000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbb435d0> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb0c3e08> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-5" #48 prio=10 os_prio=0 tid=0x00007f022400b000 nid=0x6268 runnable [0x00007f025c8d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca662bf8> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca662c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca662b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RMI RenewClean-[128.195.52.77:32962]" #46 daemon prio=5 os_prio=0 tid=0x00007f0208044800 nid=0x6260 in Object.wait() [0x00007f025cad6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142) - locked <0x00000006cb2b4618> (a java.lang.ref.ReferenceQueue$Lock) at sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:536) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "QueryTranslator" #45 prio=10 os_prio=0 tid=0x00007f024410e800 nid=0x625b in Object.wait() [0x00007f02701ba000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbd8e748> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb183678> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-4" #44 prio=10 os_prio=0 tid=0x00007f0224009800 nid=0x625a runnable [0x00007f02702bc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca6634f8> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca663558> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca663460> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "QueryTranslator" #43 prio=10 os_prio=0 tid=0x00007f023c005800 nid=0x624a in Object.wait() [0x00007f02703bc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbd99d20> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb2a07f8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-3" #42 prio=10 os_prio=0 tid=0x00007f0224007800 nid=0x6249 runnable [0x00007f02704be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca68aea0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca68af00> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca68ae08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "QueryTranslator" #41 prio=10 os_prio=0 tid=0x00007f0234001800 nid=0x623e in Object.wait() [0x00007f02841e3000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbb97d10> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb238798> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nioEventLoopGroup-5-2" #40 prio=10 os_prio=0 tid=0x00007f0224006800 nid=0x623d runnable [0x00007f02842e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca68b7a0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca68b800> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca68b708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "QueryTranslator" #39 prio=10 os_prio=0 tid=0x00007f02200b6800 nid=0x6237 in Object.wait() [0x00007f02853e6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) - locked <0x00000006cbb098a8> (a org.apache.hyracks.ipc.api.RPCInterface$Request) at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:118) at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:147) at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:2510) at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:2405) at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:2384) at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:381) at org.apache.asterix.api.http.server.QueryServiceServlet.executeStatement(QueryServiceServlet.java:530) at org.apache.asterix.api.http.server.QueryServiceServlet.handleRequest(QueryServiceServlet.java:489) at org.apache.asterix.api.http.server.QueryServiceServlet.post(QueryServiceServlet.java:111) at org.apache.hyracks.http.server.AbstractServlet.handle(AbstractServlet.java:92) at org.apache.hyracks.http.server.HttpRequestHandler.handle(HttpRequestHandler.java:71) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:56) at org.apache.hyracks.http.server.HttpRequestHandler.call(HttpRequestHandler.java:37) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006cb0ca2a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "threadDeathWatcher-6-1" #38 daemon prio=1 os_prio=0 tid=0x00007f02200ae000 nid=0x6236 waiting on condition [0x00007f02864e9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:150) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "nioEventLoopGroup-5-1" #37 prio=10 os_prio=0 tid=0x00007f0224003800 nid=0x6235 runnable [0x00007f02865ea000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca6b6068> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca6b9bf0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca6b5fc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RMI Scheduler(0)" #32 daemon prio=5 os_prio=0 tid=0x00007f0208009800 nid=0x6220 waiting on condition [0x00007f0286cef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca784248> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Worker:ClusterController" #11 daemon prio=10 os_prio=0 tid=0x00007f02a4dd4800 nid=0x61f3 waiting on condition [0x00007f02875f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006c9a447c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hyracks.control.common.work.WorkQueue$WorkerThread.run(WorkQueue.java:114) Locked ownable synchronizers: - None "Executor-1:ClusterController" #23 daemon prio=5 os_prio=0 tid=0x00007f02a4dd2800 nid=0x61f2 waiting on condition [0x00007f02876f9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hyracks.control.common.dataset.ResultStateSweeper.run(ResultStateSweeper.java:44) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000006ca8a9b58> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Thread-3" #22 prio=5 os_prio=0 tid=0x00007f02a4dcf000 nid=0x61f1 waiting on condition [0x00007f02877fa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006ca8a98a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.asterix.hyracks.bootstrap.ClusterWorkExecutor.run(ClusterWorkExecutor.java:45) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "nioEventLoopGroup-4-1" #21 prio=10 os_prio=0 tid=0x00007f02a4dc2800 nid=0x61f0 runnable [0x00007f02878fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006ca51f3a8> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ca6bb248> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006ca51f310> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "GC Daemon" #20 daemon prio=2 os_prio=0 tid=0x00007f02a4cbf800 nid=0x61ef in Object.wait() [0x00007f02879fc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at sun.misc.GC$Daemon.run(GC.java:117) - locked <0x00000006ca79aff8> (a sun.misc.GC$LatencyLock) Locked ownable synchronizers: - None "RMI Reaper" #19 prio=5 os_prio=0 tid=0x00007f02a4cbd800 nid=0x61ee in Object.wait() [0x00007f0287afd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142) - locked <0x00000006ca783710> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158) at sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:351) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RMI TCP Accept-0" #18 daemon prio=5 os_prio=0 tid=0x00007f02a4cb2000 nid=0x61ed runnable [0x00007f0287bfe000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:389) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:361) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "ActiveNotificationHandler" #17 prio=5 os_prio=0 tid=0x00007f02a4bd9800 nid=0x61ec waiting on condition [0x00007f028c1bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006c9d52168> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hyracks.api.util.SingleThreadEventProcessor.run(SingleThreadEventProcessor.java:49) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "IPC Network Listener Thread [/0:0:0:0:0:0:0:0:58476]" #16 daemon prio=10 os_prio=0 tid=0x00007f02a4b48000 nid=0x61eb runnable [0x00007f028c2bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006c9d51598> (a sun.nio.ch.Util$2) - locked <0x00000006c9d51588> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006c9d4eef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:216) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196) Locked ownable synchronizers: - None "nioEventLoopGroup-2-1" #15 prio=10 os_prio=0 tid=0x00007f02a4b27000 nid=0x61ea runnable [0x00007f028c3be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006c9aa6b68> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006c9aa5300> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006c9aa6ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:759) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:400) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "IPC Network Listener Thread [/128.195.52.77:1098]" #14 daemon prio=10 os_prio=0 tid=0x00007f02a4ae3800 nid=0x61e9 runnable [0x00007f028c4bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006c9a82a58> (a sun.nio.ch.Util$2) - locked <0x00000006c9a82a48> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006c9a82748> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:216) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196) Locked ownable synchronizers: - None "IPC Network Listener Thread [/0:0:0:0:0:0:0:0:1099]" #13 daemon prio=10 os_prio=0 tid=0x00007f02a4ae2000 nid=0x61e8 runnable [0x00007f028c5c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006c9a820c0> (a sun.nio.ch.Util$2) - locked <0x00000006c9a820b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006c9a81d68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:216) at org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196) Locked ownable synchronizers: - None "Timer-0" #12 daemon prio=5 os_prio=0 tid=0x00007f02a47c6800 nid=0x61e4 in Object.wait() [0x00007f028cac1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000006c9a316f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Service Thread" #8 daemon prio=9 os_prio=0 tid=0x00007f02a40cd000 nid=0x61e2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007f02a40b7000 nid=0x61e1 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007f02a40b5800 nid=0x61e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007f02a40b2800 nid=0x61df waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f02a40b0800 nid=0x61de runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f02a4080800 nid=0x61dd in Object.wait() [0x00007f028faf9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142) - locked <0x00000006c9883f88> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f02a407c800 nid=0x61dc in Object.wait() [0x00007f028fbfa000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157) - locked <0x00000006c9883fc8> (a java.lang.ref.Reference$Lock) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f02a400e000 nid=0x61d6 waiting on condition [0x00007f02acd35000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hyracks.control.cc.CCDriver.main(CCDriver.java:50) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f02a4077800 nid=0x61db runnable "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f02a4023000 nid=0x61d7 runnable "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f02a4025000 nid=0x61d8 runnable "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f02a4027000 nid=0x61d9 runnable "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f02a4028800 nid=0x61da runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f02a40cf800 nid=0x61e3 waiting on condition JNI global references: 594 {code} -- This message was sent by Atlassian JIRA (v7.6.3#76005)