beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Daniel Halperin (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (BEAM-1980) Seeming deadlock using Apex with relatively small data
Date Fri, 14 Apr 2017 22:13:41 GMT

    [ https://issues.apache.org/jira/browse/BEAM-1980?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15969608#comment-15969608 ] 

Daniel Halperin commented on BEAM-1980:
---------------------------------------

Just repro'ed; after 5 minutes the output is at the same place (undeployed container [2]) and the output file has not been created. So it seems stuck while processing.

jstack:

{code}
2017-04-14 15:08:34
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.112-b16 mixed mode):

"Attach Listener" #121 daemon prio=9 os_prio=31 tid=0x00007f91a9140800 nid=0x9007 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"StorageHelper-2-1" #120 prio=5 os_prio=31 tid=0x00007f91a7193000 nid=0x9a03 waiting on condition [0x000070000dac0000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cd1cf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"ServerHelper-1-1" #119 prio=5 os_prio=31 tid=0x00007f91a5a85800 nid=0x9803 waiting on condition [0x000070000d9bd000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cde730> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"9/SumTeamScores/GroupByKey:ApexGroupByKeyOperator" #118 prio=5 os_prio=31 tid=0x00007f91a70d9800 nid=0x9603 waiting on condition [0x000070000d8ba000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"10/SumTeamScores/ParDo(WriteWindowedFiles)/ParMultiDo(WriteWindowedFiles):ApexParDoOperator" #117 prio=5 os_prio=31 tid=0x00007f91a8811800 nid=0x9403 waiting on condition [0x000070000d7b7000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"3/SetTimestamps/ParMultiDo(SetTimestamps):ApexParDoOperator" #115 prio=5 os_prio=31 tid=0x00007f91a8860800 nid=0x8e03 waiting on condition [0x000070000d4ae000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"5/SumTeamScores/ParDo(KeyScoreByTeam)/ParMultiDo(KeyScoreByTeam):ApexParDoOperator" #114 prio=5 os_prio=31 tid=0x00007f91a8809000 nid=0x8c03 waiting on condition [0x000070000d3ab000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"4/FixedWindows/Window.Assign:ApexProcessFnOperator" #112 prio=5 os_prio=31 tid=0x00007f91a6a66800 nid=0x8a03 waiting on condition [0x000070000d2a8000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"6/SumTeamScores/Combine.perKey(SumInteger)/GroupByKey:ApexGroupByKeyOperator" #111 prio=5 os_prio=31 tid=0x00007f91a6a89800 nid=0x8803 waiting on condition [0x000070000d1a5000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"7/SumTeamScores/Combine.perKey(SumInteger)/Combine.GroupedValues/ParDo(Anonymous)/ParMultiDo(Anonymous):ApexParDoOperator" #109 prio=5 os_prio=31 tid=0x00007f91a5954800 nid=0x8603 waiting on condition [0x000070000d0a2000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"8/SumTeamScores/ParDo(KeyByWindow)/ParMultiDo(KeyByWindow):ApexParDoOperator" #108 prio=5 os_prio=31 tid=0x00007f91a5b18800 nid=0x1307 waiting on condition [0x000070000ce9c000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)

"pool-1-thread-4" #106 prio=5 os_prio=31 tid=0x00007f91a682e800 nid=0x8203 waiting on condition [0x000070000cd99000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0756910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"pool-1-thread-3" #107 prio=5 os_prio=31 tid=0x00007f91a682e000 nid=0x8003 waiting on condition [0x000070000cc96000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0756910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"pool-1-thread-2" #104 prio=5 os_prio=31 tid=0x00007f91a6c56800 nid=0x7e03 waiting on condition [0x000070000cb93000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0756910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"pool-1-thread-1" #105 prio=5 os_prio=31 tid=0x00007f91a680a000 nid=0x1007 waiting on condition [0x000070000ca90000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0756910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"container-9" #103 prio=5 os_prio=31 tid=0x00007f91a5be1000 nid=0x7c03 in Object.wait() [0x000070000c98d000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0760568> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-9" #102 daemon prio=5 os_prio=31 tid=0x00007f91a5b8e800 nid=0x7a03 waiting on condition [0x000070000c88a000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0760d30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-8" #101 prio=5 os_prio=31 tid=0x00007f91a665e000 nid=0x7803 in Object.wait() [0x000070000c787000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c07735d8> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-8" #100 daemon prio=5 os_prio=31 tid=0x00007f91a7350800 nid=0x7603 waiting on condition [0x000070000c684000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0778838> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-7" #99 prio=5 os_prio=31 tid=0x00007f91a5bcf000 nid=0x7403 in Object.wait() [0x000070000c581000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0cbc480> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-7" #98 daemon prio=5 os_prio=31 tid=0x00007f91a7252800 nid=0x7203 waiting on condition [0x000070000c47e000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cd19e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-6" #97 prio=5 os_prio=31 tid=0x00007f91a6bdb000 nid=0x7003 in Object.wait() [0x000070000c37b000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0cda378> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-6" #96 daemon prio=5 os_prio=31 tid=0x00007f91a665d000 nid=0x6e03 waiting on condition [0x000070000c278000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cdcb58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-5" #95 prio=5 os_prio=31 tid=0x00007f91a6663000 nid=0x6c03 in Object.wait() [0x000070000c175000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0cd6e68> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-5" #94 daemon prio=5 os_prio=31 tid=0x00007f91a67f7000 nid=0x6a03 waiting on condition [0x000070000c072000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cd9ef8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-4" #93 prio=5 os_prio=31 tid=0x00007f91a67f6000 nid=0x6803 in Object.wait() [0x000070000bf6f000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0749548> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-4" #92 daemon prio=5 os_prio=31 tid=0x00007f91a72e9000 nid=0x6603 waiting on condition [0x000070000be6c000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c074ea18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-3" #91 prio=5 os_prio=31 tid=0x00007f91a6680000 nid=0x6403 in Object.wait() [0x000070000bd69000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c05fd418> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-3" #90 daemon prio=5 os_prio=31 tid=0x00007f91a667f800 nid=0x6203 waiting on condition [0x000070000bc66000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0603d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-2" #89 prio=5 os_prio=31 tid=0x00007f91a5bd1800 nid=0x6003 in Object.wait() [0x000070000bb63000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0cd4bc0> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-2" #88 daemon prio=5 os_prio=31 tid=0x00007f91a6711000 nid=0x5e03 waiting on condition [0x000070000ba60000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cd69e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-1" #87 prio=5 os_prio=31 tid=0x00007f91a6001800 nid=0x5c03 in Object.wait() [0x000070000b95d000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c0cc2758> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-1" #86 daemon prio=5 os_prio=31 tid=0x00007f91a7252000 nid=0x5a03 waiting on condition [0x000070000b85a000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0cd4740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"container-0" #85 prio=5 os_prio=31 tid=0x00007f91a72ea000 nid=0x5803 in Object.wait() [0x000070000b757000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
	- locked <0x00000006c05c8260> (a java.lang.Object)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
	at java.lang.Thread.run(Thread.java:745)

"Dispatcher-0" #84 daemon prio=5 os_prio=31 tid=0x00007f91a7378000 nid=0x5603 waiting on condition [0x000070000b654000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0c6ecd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
	at java.lang.Thread.run(Thread.java:745)

"DestroyJavaVM" #83 prio=5 os_prio=31 tid=0x00007f91a68b8800 nid=0x1c03 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"master" #82 prio=5 os_prio=31 tid=0x00007f91a6795800 nid=0x5403 waiting on condition [0x000070000b551000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at com.datatorrent.stram.StramLocalCluster.run(StramLocalCluster.java:499)
	at com.datatorrent.stram.StramLocalCluster.run(StramLocalCluster.java:438)
	at java.lang.Thread.run(Thread.java:745)

"ProcessWideEventLoop" #81 prio=5 os_prio=31 tid=0x00007f91a5b08000 nid=0x521b runnable [0x000070000b44e000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
	at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
	at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x00000006c0c735c8> (a com.datatorrent.netlet.OptimizedEventLoop$SelectedSelectionKeySet)
	- locked <0x00000006c09a4c68> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000006c0c73508> (a sun.nio.ch.KQueueSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
	at com.datatorrent.netlet.OptimizedEventLoop.runEventLoop(OptimizedEventLoop.java:185)
	at com.datatorrent.netlet.OptimizedEventLoop.runEventLoop(OptimizedEventLoop.java:157)
	at com.datatorrent.netlet.DefaultEventLoop.run(DefaultEventLoop.java:156)
	at java.lang.Thread.run(Thread.java:745)

"process reaper" #10 daemon prio=10 os_prio=31 tid=0x00007f91a5c26800 nid=0x5007 waiting on condition [0x000070000b34b000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c0c65430> (a java.util.concurrent.SynchronousQueue$TransferStack)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
	at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"Service Thread" #8 daemon prio=9 os_prio=31 tid=0x00007f91a585c800 nid=0x4903 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f91a5851800 nid=0x4703 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f91a5846800 nid=0x4503 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f91a5845000 nid=0x4303 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f91a5850800 nid=0x4103 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f91a5837800 nid=0x3103 in Object.wait() [0x000070000ad0e000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000006c001c8f0> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
	- locked <0x00000006c001c8f0> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f91a6805000 nid=0x2f03 in Object.wait() [0x000070000ac0b000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000006c000cba0> (a java.lang.ref.Reference$Lock)
	at java.lang.Object.wait(Object.java:502)
	at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
	- locked <0x00000006c000cba0> (a java.lang.ref.Reference$Lock)
	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"VM Thread" os_prio=31 tid=0x00007f91a5835000 nid=0x2d03 runnable 

"GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f91a580b800 nid=0x2503 runnable 

"GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f91a580c800 nid=0x2703 runnable 

"GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f91a580d000 nid=0x2903 runnable 

"GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f91a580d800 nid=0x2b03 runnable 

"VM Periodic Task Thread" os_prio=31 tid=0x00007f91a700a000 nid=0x4b03 waiting on condition 

JNI global references: 283
{code}

> Seeming deadlock using Apex with relatively small data
> ------------------------------------------------------
>
>                 Key: BEAM-1980
>                 URL: https://issues.apache.org/jira/browse/BEAM-1980
>             Project: Beam
>          Issue Type: Bug
>          Components: runner-apex
>            Reporter: Daniel Halperin
>            Assignee: Thomas Weise
>             Fix For: First stable release
>
>
> I'm running the "beam portability demo" at https://github.com/dhalperi/beam-portability-demo/tree/apex
> Made a very small input file:
> {code}
> gsutil cat gs://apache-beam-demo/data2/small-game.csv | head -n 100000 > tiny.csv
> {code}
> Ran the job in embedded mode using an Apex fat-jar from the pom in that branch (and adding in {{slf4j-jdk14.jar}} for debugging info):
> {code}
> java -cp ~/.m2/repository/org/slf4j/slf4j-jdk14/1.7.14/slf4j-jdk14-1.7.14.jar:target/portability-demo-bundled-apex.jar demo.HourlyTeamScore --runner=ApexRunner --outputPrefix=gs://clouddfe-dhalperi/output/apex --input=tiny.csv
> {code}
> A good run takes O(25 seconds):
> {code}
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in [jar:file:/Users/dhalperi/.m2/repository/org/slf4j/slf4j-jdk14/1.7.14/slf4j-jdk14-1.7.14.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in [jar:file:/Users/dhalperi/beam-portability-demo/target/portability-demo-bundled-apex.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
> SLF4J: Actual binding is of type [org.slf4j.impl.JDK14LoggerFactory]
> log4j:WARN No appenders could be found for logger (org.apache.commons.beanutils.converters.BooleanConverter).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
> Apr 14, 2017 1:20:55 PM com.datatorrent.common.util.AsyncFSStorageAgent save
> INFO: using /var/folders/7r/s0gg2qb11jz4g4pcb8n15gkc009j1y/T/chkp8074838277485202831 as the basepath for checkpointing.
> Apr 14, 2017 1:20:56 PM com.datatorrent.bufferserver.storage.DiskStorage <init>
> INFO: using /var/folders/7r/s0gg2qb11jz4g4pcb8n15gkc009j1y/T as the basepath for spooling.
> Apr 14, 2017 1:20:56 PM com.datatorrent.bufferserver.server.Server registered
> INFO: Server started listening at /0:0:0:0:0:0:0:0:61087
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster <init>
> INFO: Buffer server started: localhost:61087
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-0
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-1
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-2
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-3
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-4
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-5
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-2 msg: [container-2] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-1 msg: [container-1] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-5 msg: [container-5] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-4 msg: [container-4] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-0 msg: [container-0] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-7
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-7 msg: [container-7] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-6
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-3 msg: [container-3] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-6 msg: [container-6] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-8
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-8 msg: [container-8] Entering heartbeat loop..
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-9
> Apr 14, 2017 1:20:56 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-9 msg: [container-9] Entering heartbeat loop..
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=4,name=FixedWindows/Window.Assign,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=inputPort,streamId=stream1,sourceNodeId=3,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=outputPort,streamId=stream7,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=6,name=SumTeamScores/Combine.perKey(SumInteger)/GroupByKey,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream4,sourceNodeId=5,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream0,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=2,name=ParseGameEvent/ParMultiDo(ParseEvent),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream6,sourceNodeId=1,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream2,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=6.output.6, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/5.output.5, windowId=ffffffffffffffff, type=stream4/6.input, upstreamIdentifier=5.output.5, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=4.outputPort.4, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=2.output.2, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/3.output.3, windowId=ffffffffffffffff, type=stream1/4.inputPort, upstreamIdentifier=3.output.3, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/1.output.1, windowId=ffffffffffffffff, type=stream6/2.input, upstreamIdentifier=1.output.1, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=10,name=SumTeamScores/ParDo(WriteWindowedFiles)/ParMultiDo(WriteWindowedFiles),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream3,sourceNodeId=9,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=8,name=SumTeamScores/ParDo(KeyByWindow)/ParMultiDo(KeyByWindow),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream5,sourceNodeId=7,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream8,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=5,name=SumTeamScores/ParDo(KeyScoreByTeam)/ParMultiDo(KeyScoreByTeam),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream7,sourceNodeId=4,sourcePortName=outputPort,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream4,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=3,name=SetTimestamps/ParMultiDo(SetTimestamps),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream2,sourceNodeId=2,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream1,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=7,name=SumTeamScores/Combine.perKey(SumInteger)/Combine.GroupedValues/ParDo(Anonymous)/ParMultiDo(Anonymous),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream0,sourceNodeId=6,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream5,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=1,name=TextIO.Read/Read,type=INPUT,checkpoint={ffffffffffffffff, 0, 0},inputs=[],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream6,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=9,name=SumTeamScores/GroupByKey,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream8,sourceNodeId=8,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream3,bufferServer=localhost]]]]
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=8.output.8, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/7.output.7, windowId=ffffffffffffffff, type=stream5/8.input, upstreamIdentifier=7.output.7, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/9.output.9, windowId=ffffffffffffffff, type=stream3/10.input, upstreamIdentifier=9.output.9, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=5.output.5, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/4.outputPort.4, windowId=ffffffffffffffff, type=stream7/5.input, upstreamIdentifier=4.outputPort.4, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.stram.engine.WindowGenerator activate
> INFO: Catching up from 1492201255500 to 1492201257769
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/2.output.2, windowId=ffffffffffffffff, type=stream2/3.input, upstreamIdentifier=2.output.2, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=3.output.3, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=9.output.9, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/8.output.8, windowId=ffffffffffffffff, type=stream8/9.input, upstreamIdentifier=8.output.8, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61087/6.output.6, windowId=ffffffffffffffff, type=stream0/7.input, upstreamIdentifier=6.output.6, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=7.output.7, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=1.output.1, windowId=ffffffffffffffff}
> Apr 14, 2017 1:20:57 PM org.apache.beam.sdk.io.FileBasedSource expandFilePattern
> INFO: Matched 1 files for pattern tiny.csv
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:20:58 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:01 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [1]
> Apr 14, 2017 1:21:01 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:03 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [2]
> Apr 14, 2017 1:21:03 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:07 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [5]
> Apr 14, 2017 1:21:07 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:07 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [3]
> Apr 14, 2017 1:21:07 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:08 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [4]
> Apr 14, 2017 1:21:08 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [6]
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [8]
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [9]
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [7]
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:20 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processEvents
> INFO: Removing inactive operators at window 58f12f270000000b [PTOperator[id=1,name=TextIO.Read/Read]]
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-9
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-0
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-9 msg: [container-9] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-0 msg: [container-0] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-9 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-0 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-7
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-7 msg: [container-7] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-7 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-4
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-6
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-6 msg: [container-6] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-4 msg: [container-4] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-6 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-4 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-5
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: Heartbeat for unknown operator 10 (container container-1)
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting shutdown for container container-1
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-5 msg: [container-5] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [10]
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-5 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-1 msg: [container-1] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-1 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-2
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-3
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-2 msg: [container-2] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-3 msg: [container-3] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-2 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-3 terminating.
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StreamingContainerManager processHeartbeat
> INFO: requesting idle shutdown for container container-8
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Received shutdown request
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-8 msg: [container-8] Exiting heartbeat loop..
> Apr 14, 2017 1:21:21 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Container container-8 terminating.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.StreamingContainerManager processEvents
> INFO: Removing inactive operators at window 58f12f270000000c [PTOperator[id=9,name=SumTeamScores/GroupByKey], PTOperator[id=4,name=FixedWindows/Window.Assign], PTOperator[id=5,name=SumTeamScores/ParDo(KeyScoreByTeam)/ParMultiDo(KeyScoreByTeam)], PTOperator[id=3,name=SetTimestamps/ParMultiDo(SetTimestamps)], PTOperator[id=2,name=ParseGameEvent/ParMultiDo(ParseEvent)], PTOperator[id=6,name=SumTeamScores/Combine.perKey(SumInteger)/GroupByKey], PTOperator[id=8,name=SumTeamScores/ParDo(KeyByWindow)/ParMultiDo(KeyByWindow)], PTOperator[id=7,name=SumTeamScores/Combine.perKey(SumInteger)/Combine.GroupedValues/ParDo(Anonymous)/ParMultiDo(Anonymous)]]
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:21:22 PM com.datatorrent.stram.StramLocalCluster run
> INFO: Application finished.
> Apr 14, 2017 1:21:22 PM com.datatorrent.bufferserver.server.Server unregistered
> INFO: Server stopped listening at /0:0:0:0:0:0:0:0:61087
> {code}
> However I got a seemingly stalled run, for which I captured a {{jstack}} after about 3 minutes:
> {code}
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in [jar:file:/Users/dhalperi/.m2/repository/org/slf4j/slf4j-jdk14/1.7.14/slf4j-jdk14-1.7.14.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in [jar:file:/Users/dhalperi/beam-portability-demo/target/portability-demo-bundled-apex.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
> SLF4J: Actual binding is of type [org.slf4j.impl.JDK14LoggerFactory]
> log4j:WARN No appenders could be found for logger (org.apache.commons.beanutils.converters.BooleanConverter).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
> Apr 14, 2017 1:16:52 PM com.datatorrent.common.util.AsyncFSStorageAgent save
> INFO: using /var/folders/7r/s0gg2qb11jz4g4pcb8n15gkc009j1y/T/chkp496589819849283689 as the basepath for checkpointing.
> Apr 14, 2017 1:16:53 PM com.datatorrent.bufferserver.storage.DiskStorage <init>
> INFO: using /var/folders/7r/s0gg2qb11jz4g4pcb8n15gkc009j1y/T as the basepath for spooling.
> Apr 14, 2017 1:16:53 PM com.datatorrent.bufferserver.server.Server registered
> INFO: Server started listening at /0:0:0:0:0:0:0:0:61042
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster <init>
> INFO: Buffer server started: localhost:61042
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-0
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-1
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-2
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-3
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-4
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-3 msg: [container-3] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-2 msg: [container-2] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-1 msg: [container-1] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-4 msg: [container-4] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-0 msg: [container-0] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-5
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-5 msg: [container-5] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-6
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-7
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-8
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-7 msg: [container-7] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-6 msg: [container-6] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-8 msg: [container-8] Entering heartbeat loop..
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher run
> INFO: Started container container-9
> Apr 14, 2017 1:16:53 PM com.datatorrent.stram.StramLocalCluster$UmbilicalProtocolLocalImpl log
> INFO: container-9 msg: [container-9] Entering heartbeat loop..
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer heartbeatLoop
> INFO: Waiting for pending request.
> Apr 14, 2017 1:16:54 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=4,name=FixedWindows/Window.Assign,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=inputPort,streamId=stream1,sourceNodeId=3,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=outputPort,streamId=stream7,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:54 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=4.outputPort.4, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:54 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/3.output.3, windowId=ffffffffffffffff, type=stream1/4.inputPort, upstreamIdentifier=3.output.3, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=3,name=SetTimestamps/ParMultiDo(SetTimestamps),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream2,sourceNodeId=2,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream1,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=9,name=SumTeamScores/GroupByKey,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream8,sourceNodeId=8,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream3,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=5,name=SumTeamScores/ParDo(KeyScoreByTeam)/ParMultiDo(KeyScoreByTeam),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream7,sourceNodeId=4,sourcePortName=outputPort,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream4,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=1,name=TextIO.Read/Read,type=INPUT,checkpoint={ffffffffffffffff, 0, 0},inputs=[],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream6,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=6,name=SumTeamScores/Combine.perKey(SumInteger)/GroupByKey,type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream4,sourceNodeId=5,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream0,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=2,name=ParseGameEvent/ParMultiDo(ParseEvent),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream6,sourceNodeId=1,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream2,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=7,name=SumTeamScores/Combine.perKey(SumInteger)/Combine.GroupedValues/ParDo(Anonymous)/ParMultiDo(Anonymous),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream0,sourceNodeId=6,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream5,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=8,name=SumTeamScores/ParDo(KeyByWindow)/ParMultiDo(KeyByWindow),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream5,sourceNodeId=7,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[OperatorDeployInfo.OutputDeployInfo[portName=output,streamId=stream8,bufferServer=localhost]]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Deploy request: [OperatorDeployInfo[id=10,name=SumTeamScores/ParDo(WriteWindowedFiles)/ParMultiDo(WriteWindowedFiles),type=GENERIC,checkpoint={ffffffffffffffff, 0, 0},inputs=[OperatorDeployInfo.InputDeployInfo[portName=input,streamId=stream3,sourceNodeId=9,sourcePortName=output,locality=<null>,partitionMask=0,partitionKeys=<null>]],outputs=[]]]
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/2.output.2, windowId=ffffffffffffffff, type=stream2/3.input, upstreamIdentifier=2.output.2, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=3.output.3, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.engine.WindowGenerator activate
> INFO: Catching up from 1492201012500 to 1492201015057
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=9.output.9, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/8.output.8, windowId=ffffffffffffffff, type=stream8/9.input, upstreamIdentifier=8.output.8, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/1.output.1, windowId=ffffffffffffffff, type=stream6/2.input, upstreamIdentifier=1.output.1, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=2.output.2, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=6.output.6, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/5.output.5, windowId=ffffffffffffffff, type=stream4/6.input, upstreamIdentifier=5.output.5, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/7.output.7, windowId=ffffffffffffffff, type=stream5/8.input, upstreamIdentifier=7.output.7, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=8.output.8, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=1.output.1, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=5.output.5, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/4.outputPort.4, windowId=ffffffffffffffff, type=stream7/5.input, upstreamIdentifier=4.outputPort.4, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/9.output.9, windowId=ffffffffffffffff, type=stream3/10.input, upstreamIdentifier=9.output.9, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received subscriber request: SubscribeRequestTuple{version=1.0, identifier=tcp://localhost:61042/6.output.6, windowId=ffffffffffffffff, type=stream0/7.input, upstreamIdentifier=6.output.6, mask=0, partitions=null, bufferSize=0}
> Apr 14, 2017 1:16:55 PM com.datatorrent.bufferserver.server.Server$UnidentifiedClient onMessage
> INFO: Received publisher request: PublishRequestTuple{version=1.0, identifier=7.output.7, windowId=ffffffffffffffff}
> Apr 14, 2017 1:16:55 PM org.apache.beam.sdk.io.FileBasedSource expandFilePattern
> INFO: Matched 1 files for pattern tiny.csv
> Apr 14, 2017 1:16:55 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:56 PM com.datatorrent.stram.Journal write
> WARNING: Journal output stream is null. Skipping write to the WAL.
> Apr 14, 2017 1:16:58 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [1]
> Apr 14, 2017 1:16:58 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> Apr 14, 2017 1:17:01 PM com.datatorrent.stram.engine.StreamingContainer processHeartbeatResponse
> INFO: Undeploy request: [2]
> Apr 14, 2017 1:17:01 PM com.datatorrent.stram.engine.StreamingContainer undeploy
> INFO: Undeploy complete.
> {code}
> JSTACK:
> {code}
> ➜  beam-portability-demo git:(apex) ✗ jstack 84028
> 2017-04-14 13:19:55
> Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.112-b16 mixed mode):
> "Attach Listener" #121 daemon prio=9 os_prio=31 tid=0x00007f8a0d800800 nid=0x4f0b waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "StorageHelper-2-1" #120 prio=5 os_prio=31 tid=0x00007f8a0d21a000 nid=0x1207 waiting on condition [0x0000700000801000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0ddba38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "7/SumTeamScores/Combine.perKey(SumInteger)/Combine.GroupedValues/ParDo(Anonymous)/ParMultiDo(Anonymous):ApexParDoOperator" #119 prio=5 os_prio=31 tid=0x00007f8a0b413000 nid=0x9b03 waiting on condition [0x0000700003d4e000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "10/SumTeamScores/ParDo(WriteWindowedFiles)/ParMultiDo(WriteWindowedFiles):ApexParDoOperator" #117 prio=5 os_prio=31 tid=0x00007f8a0b3f9800 nid=0x9703 waiting on condition [0x0000700003b48000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "5/SumTeamScores/ParDo(KeyScoreByTeam)/ParMultiDo(KeyScoreByTeam):ApexParDoOperator" #116 prio=5 os_prio=31 tid=0x00007f8a0b3f9000 nid=0x9503 waiting on condition [0x0000700003a45000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "8/SumTeamScores/ParDo(KeyByWindow)/ParMultiDo(KeyByWindow):ApexParDoOperator" #114 prio=5 os_prio=31 tid=0x00007f8a0a7f7000 nid=0x9103 waiting on condition [0x000070000383f000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "6/SumTeamScores/Combine.perKey(SumInteger)/GroupByKey:ApexGroupByKeyOperator" #113 prio=5 os_prio=31 tid=0x00007f8a0abfc800 nid=0x8f03 waiting on condition [0x000070000373c000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "9/SumTeamScores/GroupByKey:ApexGroupByKeyOperator" #111 prio=5 os_prio=31 tid=0x00007f8a0a6ee800 nid=0x8b03 waiting on condition [0x0000700003536000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "3/SetTimestamps/ParMultiDo(SetTimestamps):ApexParDoOperator" #110 prio=5 os_prio=31 tid=0x00007f8a0aaff000 nid=0x8903 waiting on condition [0x0000700003433000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "ServerHelper-1-1" #109 prio=5 os_prio=31 tid=0x00007f8a0bd25800 nid=0x8703 waiting on condition [0x0000700003330000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0fcc600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "4/FixedWindows/Window.Assign:ApexProcessFnOperator" #108 prio=5 os_prio=31 tid=0x00007f8a0cc85800 nid=0x8503 waiting on condition [0x000070000322d000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:601)
> 	at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1407)
> "pool-1-thread-1" #105 prio=5 os_prio=31 tid=0x00007f8a0b314000 nid=0x8303 waiting on condition [0x000070000312a000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0ce0130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "pool-1-thread-4" #107 prio=5 os_prio=31 tid=0x00007f8a0ab46000 nid=0x8103 waiting on condition [0x0000700003027000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0ce0130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "pool-1-thread-2" #106 prio=5 os_prio=31 tid=0x00007f8a0ab10000 nid=0x7f03 waiting on condition [0x0000700002f24000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0ce0130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "pool-1-thread-3" #104 prio=5 os_prio=31 tid=0x00007f8a0bc4e800 nid=0x7d03 waiting on condition [0x0000700002e21000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0ce0130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-9" #103 prio=5 os_prio=31 tid=0x00007f8a0bc64000 nid=0x7b03 in Object.wait() [0x0000700002d1e000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0ce92e0> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-9" #102 daemon prio=5 os_prio=31 tid=0x00007f8a0bd0c800 nid=0x7903 waiting on condition [0x0000700002c1b000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0cee530> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-8" #101 prio=5 os_prio=31 tid=0x00007f8a0b31b000 nid=0x7703 in Object.wait() [0x0000700002b18000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0d52250> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-8" #100 daemon prio=5 os_prio=31 tid=0x00007f8a0ab44000 nid=0x7503 waiting on condition [0x0000700002a15000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0d36a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-7" #99 prio=5 os_prio=31 tid=0x00007f8a0ab43800 nid=0x7303 in Object.wait() [0x0000700002912000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0d07a98> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-7" #98 daemon prio=5 os_prio=31 tid=0x00007f8a0ab42800 nid=0x7103 waiting on condition [0x000070000280f000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0d0ab08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-6" #97 prio=5 os_prio=31 tid=0x00007f8a0ab40000 nid=0x6f03 in Object.wait() [0x000070000270c000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0d77790> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-6" #96 daemon prio=5 os_prio=31 tid=0x00007f8a0ab3f000 nid=0x6d03 waiting on condition [0x0000700002609000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0d67200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-5" #95 prio=5 os_prio=31 tid=0x00007f8a0a5c5800 nid=0x6b03 in Object.wait() [0x0000700002506000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0dd51f0> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-5" #94 daemon prio=5 os_prio=31 tid=0x00007f8a0a565800 nid=0x6903 waiting on condition [0x0000700002403000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0f21998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-4" #93 prio=5 os_prio=31 tid=0x00007f8a0b40e800 nid=0x6703 in Object.wait() [0x0000700002300000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0f5d0b8> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-4" #92 daemon prio=5 os_prio=31 tid=0x00007f8a0b3c6000 nid=0x6503 waiting on condition [0x00007000021fd000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0f60db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-3" #91 prio=5 os_prio=31 tid=0x00007f8a0ab4b800 nid=0x6303 in Object.wait() [0x00007000020fa000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0f96a08> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-3" #90 daemon prio=5 os_prio=31 tid=0x00007f8a0b3cc800 nid=0x6103 waiting on condition [0x0000700001ff7000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0f9afe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-2" #89 prio=5 os_prio=31 tid=0x00007f8a0b3cc000 nid=0x5f03 in Object.wait() [0x0000700001ef4000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c0fc44b8> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-2" #88 daemon prio=5 os_prio=31 tid=0x00007f8a0ab4b000 nid=0x5d03 waiting on condition [0x0000700001df1000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c0fcb2d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-1" #87 prio=5 os_prio=31 tid=0x00007f8a0ab11800 nid=0x5b03 in Object.wait() [0x0000700001cee000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c12df300> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-1" #86 daemon prio=5 os_prio=31 tid=0x00007f8a0b3cb000 nid=0x5903 waiting on condition [0x0000700001beb000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c1127b08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "container-0" #85 prio=5 os_prio=31 tid=0x00007f8a0bbda000 nid=0x5703 in Object.wait() [0x0000700001ae8000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at com.datatorrent.stram.engine.StreamingContainer.heartbeatLoop(StreamingContainer.java:625)
> 	- locked <0x00000006c06e2d20> (a java.lang.Object)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainer.run(StramLocalCluster.java:184)
> 	at com.datatorrent.stram.StramLocalCluster$LocalStreamingContainerLauncher.run(StramLocalCluster.java:270)
> 	at java.lang.Thread.run(Thread.java:745)
> "Dispatcher-0" #84 daemon prio=5 os_prio=31 tid=0x00007f8a0b323000 nid=0x5503 waiting on condition [0x00007000019e5000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006c112abb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at net.engio.mbassy.bus.AbstractSyncAsyncMessageBus$1.run(AbstractSyncAsyncMessageBus.java:51)
> 	at java.lang.Thread.run(Thread.java:745)
> "DestroyJavaVM" #83 prio=5 os_prio=31 tid=0x00007f8a0bc3f800 nid=0x1c03 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "master" #82 prio=5 os_prio=31 tid=0x00007f8a0ab4c800 nid=0x5303 waiting on condition [0x00007000018e2000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at com.datatorrent.stram.StramLocalCluster.run(StramLocalCluster.java:499)
> 	at com.datatorrent.stram.StramLocalCluster.run(StramLocalCluster.java:438)
> 	at java.lang.Thread.run(Thread.java:745)
> "ProcessWideEventLoop" #81 prio=5 os_prio=31 tid=0x00007f8a0ab4f800 nid=0x511b runnable [0x00007000017df000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> 	at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> 	at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x00000006c0cd6f40> (a com.datatorrent.netlet.OptimizedEventLoop$SelectedSelectionKeySet)
> 	- locked <0x00000006c09a6fb8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x00000006c0cd6e90> (a sun.nio.ch.KQueueSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
> 	at com.datatorrent.netlet.OptimizedEventLoop.runEventLoop(OptimizedEventLoop.java:185)
> 	at com.datatorrent.netlet.OptimizedEventLoop.runEventLoop(OptimizedEventLoop.java:157)
> 	at com.datatorrent.netlet.DefaultEventLoop.run(DefaultEventLoop.java:156)
> 	at java.lang.Thread.run(Thread.java:745)
> "Service Thread" #8 daemon prio=9 os_prio=31 tid=0x00007f8a0a061000 nid=0x4803 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "C1 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8a0a838800 nid=0x4603 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8a0a056000 nid=0x4403 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8a0a06c000 nid=0x4203 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8a0b02c000 nid=0x4003 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8a0a835800 nid=0x3103 in Object.wait() [0x000070000109f000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x00000006c0005fe0> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x00000006c0005fe0> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
> "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8a0b804000 nid=0x2f03 in Object.wait() [0x0000700000f9c000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x00000006c000d548> (a java.lang.ref.Reference$Lock)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
> 	- locked <0x00000006c000d548> (a java.lang.ref.Reference$Lock)
> 	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
> "VM Thread" os_prio=31 tid=0x00007f8a0b801800 nid=0x2d03 runnable 
> "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8a0b000800 nid=0x2503 runnable 
> "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8a0b800800 nid=0x2703 runnable 
> "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8a0a015000 nid=0x2903 runnable 
> "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8a0a015800 nid=0x2b03 runnable 
> "VM Periodic Task Thread" os_prio=31 tid=0x00007f8a0a09e000 nid=0x4a03 waiting on condition 
> JNI global references: 332
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message